From 480e984417c17915b31e33de193ff08e118fcec9 Mon Sep 17 00:00:00 2001
From: at055612 <22818309+at055612@users.noreply.github.com>
Date: Thu, 6 Mar 2025 16:04:06 +0000
Subject: [PATCH 01/90] gh-249 Remove MDB_UNSIGNEDKEY, let CursorIterable call
mdb_cmp
There are now essentially three ways of configuring comparators
when creating a Dbi.
**null comparator**
LMDB will use its own comparator & CursorIterable will call down
to mdb_cmp for comparisons between the current cursor key and the
range start/stop key.
**provided comparator**
LMDB will use its own comparator & CursorIterable will use the
provided comparator for comparisons between the current cursor
key and the range start/stop key.
**provided comparator with nativeCb==true**
LMDB will call back to java for all comparator duties.
CursorIterable will use the same provided comparator for
comparisons between the current cursor key and the range
start/stop key.
The methods `getSignedComparator()` and `getUnsignedComparator()`
have been made public so users of this library can access them.
---
src/main/java/org/lmdbjava/BufferProxy.java | 40 +-
.../java/org/lmdbjava/ByteArrayProxy.java | 4 +-
src/main/java/org/lmdbjava/ByteBufProxy.java | 4 +-
.../java/org/lmdbjava/ByteBufferProxy.java | 4 +-
src/main/java/org/lmdbjava/Cursor.java | 4 +
.../java/org/lmdbjava/CursorIterable.java | 403 +++++++++++-------
src/main/java/org/lmdbjava/Dbi.java | 15 +-
src/main/java/org/lmdbjava/DbiFlags.java | 21 +-
.../java/org/lmdbjava/DirectBufferProxy.java | 4 +-
src/main/java/org/lmdbjava/Env.java | 21 +-
src/main/java/org/lmdbjava/Key.java | 74 ++++
src/main/java/org/lmdbjava/KeyRangeType.java | 48 +--
src/main/java/org/lmdbjava/Library.java | 2 +
.../java/org/lmdbjava/RangeComparator.java | 19 +
.../java/org/lmdbjava/ComparatorTest.java | 8 +-
.../org/lmdbjava/CursorIterablePerfTest.java | 163 +++++++
.../java/org/lmdbjava/CursorIterableTest.java | 238 ++++++++---
src/test/java/org/lmdbjava/DbiTest.java | 4 +-
src/test/java/org/lmdbjava/KeyRangeTest.java | 5 +-
src/test/java/org/lmdbjava/TestUtils.java | 2 +
20 files changed, 768 insertions(+), 315 deletions(-)
create mode 100644 src/main/java/org/lmdbjava/Key.java
create mode 100644 src/main/java/org/lmdbjava/RangeComparator.java
create mode 100644 src/test/java/org/lmdbjava/CursorIterablePerfTest.java
diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java
index e66031d2..26d9db74 100644
--- a/src/main/java/org/lmdbjava/BufferProxy.java
+++ b/src/main/java/org/lmdbjava/BufferProxy.java
@@ -16,10 +16,6 @@
package org.lmdbjava;
import static java.lang.Long.BYTES;
-import static org.lmdbjava.DbiFlags.MDB_INTEGERKEY;
-import static org.lmdbjava.DbiFlags.MDB_UNSIGNEDKEY;
-import static org.lmdbjava.MaskedFlag.isSet;
-import static org.lmdbjava.MaskedFlag.mask;
import java.util.Comparator;
import jnr.ffi.Pointer;
@@ -70,36 +66,25 @@ protected BufferProxy() {}
*/
protected abstract byte[] getBytes(T buffer);
- /**
- * Get a suitable default {@link Comparator} given the provided flags.
- *
- *
The provided comparator must strictly match the lexicographical order of keys in the native
- * LMDB database.
- *
- * @param flags for the database
- * @return a comparator that can be used (never null)
- */
- protected Comparator getComparator(DbiFlags... flags) {
- final int intFlag = mask(flags);
-
- return isSet(intFlag, MDB_INTEGERKEY) || isSet(intFlag, MDB_UNSIGNEDKEY)
- ? getUnsignedComparator()
- : getSignedComparator();
- }
-
/**
* Get a suitable default {@link Comparator} to compare numeric key values as signed.
*
+ *
+ * Note: LMDB's default comparator is unsigned so if this is used only for the {@link CursorIterable}
+ * start/stop key comparisons then its behaviour will differ from the iteration order. Use
+ * with caution.
+ *
+ *
* @return a comparator that can be used (never null)
*/
- protected abstract Comparator getSignedComparator();
+ public abstract Comparator getSignedComparator();
/**
* Get a suitable default {@link Comparator} to compare numeric key values as unsigned.
*
* @return a comparator that can be used (never null)
*/
- protected abstract Comparator getUnsignedComparator();
+ public abstract Comparator getUnsignedComparator();
/**
* Called when the MDB_val should be set to reflect the passed buffer. This buffer
@@ -140,4 +125,13 @@ protected Comparator getComparator(DbiFlags... flags) {
final KeyVal keyVal() {
return new KeyVal<>(this);
}
+
+ /**
+ * Create a new {@link Key} to hold pointers for this buffer proxy.
+ *
+ * @return a non-null key holder
+ */
+ final Key key() {
+ return new Key<>(this);
+ }
}
diff --git a/src/main/java/org/lmdbjava/ByteArrayProxy.java b/src/main/java/org/lmdbjava/ByteArrayProxy.java
index 4a22ab83..3aeba047 100644
--- a/src/main/java/org/lmdbjava/ByteArrayProxy.java
+++ b/src/main/java/org/lmdbjava/ByteArrayProxy.java
@@ -104,12 +104,12 @@ protected byte[] getBytes(final byte[] buffer) {
}
@Override
- protected Comparator getSignedComparator() {
+ public Comparator getSignedComparator() {
return signedComparator;
}
@Override
- protected Comparator getUnsignedComparator() {
+ public Comparator getUnsignedComparator() {
return unsignedComparator;
}
diff --git a/src/main/java/org/lmdbjava/ByteBufProxy.java b/src/main/java/org/lmdbjava/ByteBufProxy.java
index cac5b97b..933b52a4 100644
--- a/src/main/java/org/lmdbjava/ByteBufProxy.java
+++ b/src/main/java/org/lmdbjava/ByteBufProxy.java
@@ -114,12 +114,12 @@ protected ByteBuf allocate() {
}
@Override
- protected Comparator getSignedComparator() {
+ public Comparator getSignedComparator() {
return comparator;
}
@Override
- protected Comparator getUnsignedComparator() {
+ public Comparator getUnsignedComparator() {
return comparator;
}
diff --git a/src/main/java/org/lmdbjava/ByteBufferProxy.java b/src/main/java/org/lmdbjava/ByteBufferProxy.java
index 2b7cdf0d..1fce090a 100644
--- a/src/main/java/org/lmdbjava/ByteBufferProxy.java
+++ b/src/main/java/org/lmdbjava/ByteBufferProxy.java
@@ -182,12 +182,12 @@ protected final ByteBuffer allocate() {
}
@Override
- protected Comparator getSignedComparator() {
+ public Comparator getSignedComparator() {
return signedComparator;
}
@Override
- protected Comparator getUnsignedComparator() {
+ public Comparator getUnsignedComparator() {
return unsignedComparator;
}
diff --git a/src/main/java/org/lmdbjava/Cursor.java b/src/main/java/org/lmdbjava/Cursor.java
index d49a9bed..9070cff6 100644
--- a/src/main/java/org/lmdbjava/Cursor.java
+++ b/src/main/java/org/lmdbjava/Cursor.java
@@ -196,6 +196,10 @@ public T key() {
return kv.key();
}
+ KeyVal keyVal() {
+ return kv;
+ }
+
/**
* Position at last key/data item.
*
diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java
index 6a03bd90..6b92a9cd 100644
--- a/src/main/java/org/lmdbjava/CursorIterable.java
+++ b/src/main/java/org/lmdbjava/CursorIterable.java
@@ -21,10 +21,14 @@
import static org.lmdbjava.CursorIterable.State.REQUIRES_NEXT_OP;
import static org.lmdbjava.CursorIterable.State.TERMINATED;
import static org.lmdbjava.GetOp.MDB_SET_RANGE;
+import static org.lmdbjava.Library.LIB;
import java.util.Comparator;
import java.util.Iterator;
import java.util.NoSuchElementException;
+import java.util.Objects;
+import java.util.function.Supplier;
+import jnr.ffi.Pointer;
import org.lmdbjava.KeyRangeType.CursorOp;
import org.lmdbjava.KeyRangeType.IteratorOp;
@@ -38,185 +42,266 @@
*/
public final class CursorIterable implements Iterable>, AutoCloseable {
- private final Comparator comparator;
- private final Cursor cursor;
- private final KeyVal entry;
- private boolean iteratorReturned;
- private final KeyRange range;
- private State state = REQUIRES_INITIAL_OP;
-
- CursorIterable(
- final Txn txn, final Dbi dbi, final KeyRange range, final Comparator comparator) {
- this.cursor = dbi.openCursor(txn);
- this.range = range;
- this.comparator = comparator;
- this.entry = new KeyVal<>();
- }
-
- @Override
- public void close() {
- cursor.close();
- }
-
- /**
- * Obtain an iterator.
- *
- * As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an
- * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than
- * once. For advanced cursor control (such as being able to iterate over the same data multiple
- * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}.
- *
- * @return an iterator
- */
- @Override
- public Iterator> iterator() {
- if (iteratorReturned) {
- throw new IllegalStateException("Iterator can only be returned once");
- }
- iteratorReturned = true;
+ // private final Comparator comparator;
+ private final RangeComparator rangeComparator;
+ private final Cursor cursor;
+ private final Dbi dbi;
+ private final KeyVal entry;
+ private boolean iteratorReturned;
+ private final KeyRange range;
+ private State state = REQUIRES_INITIAL_OP;
+ private final Key startKey;
+ private final Key stopKey;
- return new Iterator>() {
- @Override
- public boolean hasNext() {
- while (state != RELEASED && state != TERMINATED) {
- update();
- }
- return state == RELEASED;
- }
+ CursorIterable(
+ final Txn txn,
+ final Dbi dbi,
+ final KeyRange range,
+ final Comparator comparator,
+ final BufferProxy proxy) {
+ this.cursor = dbi.openCursor(txn);
+ this.dbi = dbi;
+ this.range = range;
+ this.entry = new KeyVal<>();
- @Override
- public KeyVal next() {
- if (!hasNext()) {
- throw new NoSuchElementException();
+ if (comparator != null) {
+ // User supplied java-side comparator so use that
+ this.rangeComparator = createJavaRangeComparator(range, comparator, entry::key);
+ this.startKey = null;
+ this.stopKey = null;
+ } else {
+ // No java-side comparator so call down to LMDB to do the comparison
+ this.rangeComparator = createLmdbDbiComparator(txn.pointer(), dbi.pointer());
+ // Allocate buffers for use with the start/stop keys if required.
+ // Saves us copying bytes on each comparison
+ this.startKey = createKey(range.getStart(), proxy);
+ this.stopKey = createKey(range.getStop(), proxy);
}
- state = REQUIRES_NEXT_OP;
- return entry;
- }
-
- @Override
- public void remove() {
- cursor.delete();
- }
- };
- }
-
- private void executeCursorOp(final CursorOp op) {
- final boolean found;
- switch (op) {
- case FIRST:
- found = cursor.first();
- break;
- case LAST:
- found = cursor.last();
- break;
- case NEXT:
- found = cursor.next();
- break;
- case PREV:
- found = cursor.prev();
- break;
- case GET_START_KEY:
- found = cursor.get(range.getStart(), MDB_SET_RANGE);
- break;
- case GET_START_KEY_BACKWARD:
- found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last();
- break;
- default:
- throw new IllegalStateException("Unknown cursor operation");
}
- entry.setK(found ? cursor.key() : null);
- entry.setV(found ? cursor.val() : null);
- }
-
- private void executeIteratorOp() {
- final IteratorOp op =
- range.getType().iteratorOp(range.getStart(), range.getStop(), entry.key(), comparator);
- switch (op) {
- case CALL_NEXT_OP:
- executeCursorOp(range.getType().nextOp());
- state = REQUIRES_ITERATOR_OP;
- break;
- case TERMINATE:
- state = TERMINATED;
- break;
- case RELEASE:
- state = RELEASED;
- break;
- default:
- throw new IllegalStateException("Unknown operation");
+
+ private Key createKey(final T keyBuffer, final BufferProxy proxy) {
+ if (keyBuffer != null) {
+ final Key key = proxy.key();
+ key.keyIn(keyBuffer);
+ return key;
+ } else {
+ return null;
+ }
}
- }
-
- private void update() {
- switch (state) {
- case REQUIRES_INITIAL_OP:
- executeCursorOp(range.getType().initialOp());
- state = REQUIRES_ITERATOR_OP;
- break;
- case REQUIRES_NEXT_OP:
- executeCursorOp(range.getType().nextOp());
- state = REQUIRES_ITERATOR_OP;
- break;
- case REQUIRES_ITERATOR_OP:
- executeIteratorOp();
- break;
- case TERMINATED:
- break;
- default:
- throw new IllegalStateException("Unknown state");
+
+ static RangeComparator createJavaRangeComparator(
+ final KeyRange range,
+ final Comparator comparator,
+ final Supplier currentKeySupplier) {
+ final T start = range.getStart();
+ final T stop = range.getStop();
+ return new RangeComparator() {
+ @Override
+ public int compareToStartKey() {
+ return comparator.compare(currentKeySupplier.get(), start);
+ }
+
+ @Override
+ public int compareToStopKey() {
+ return comparator.compare(currentKeySupplier.get(), stop);
+ }
+ };
}
- }
- /**
- * Holder for a key and value pair.
- *
- * The same holder instance will always be returned for a given iterator. The returned keys and
- * values may change or point to different memory locations following changes in the iterator,
- * cursor or transaction.
- *
- * @param buffer type
- */
- public static final class KeyVal {
+ /**
+ * Calls down to mdb_cmp to make use of the comparator that LMDB uses for insertion order.
+ *
+ * @param txnPointer The pointer to the transaction.
+ * @param dbiPointer The pointer to the Dbi so LMDB can use the comparator of the Dbi
+ */
+ private RangeComparator createLmdbDbiComparator(
+ final Pointer txnPointer, final Pointer dbiPointer) {
+ Objects.requireNonNull(txnPointer);
+ Objects.requireNonNull(dbiPointer);
+ Objects.requireNonNull(cursor);
+
+ return new RangeComparator() {
+ @Override
+ public int compareToStartKey() {
+ return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), startKey.pointerKey());
+ }
- private T k;
- private T v;
+ @Override
+ public int compareToStopKey() {
+ return LIB.mdb_cmp(txnPointer, dbiPointer, cursor.keyVal().pointerKey(), stopKey.pointerKey());
+ }
+ };
+ }
- /** Explicitly-defined default constructor to avoid warnings. */
- public KeyVal() {}
+ @Override
+ public void close() {
+ cursor.close();
+ }
/**
- * The key.
+ * Obtain an iterator.
+ *
+ * As iteration of the returned iterator will cause movement of the underlying LMDB cursor, an
+ * {@link IllegalStateException} is thrown if an attempt is made to obtain the iterator more than
+ * once. For advanced cursor control (such as being able to iterate over the same data multiple
+ * times etc) please instead refer to {@link Dbi#openCursor(org.lmdbjava.Txn)}.
*
- * @return key
+ * @return an iterator
*/
- public T key() {
- return k;
+ @Override
+ public Iterator> iterator() {
+ if (iteratorReturned) {
+ throw new IllegalStateException("Iterator can only be returned once");
+ }
+ iteratorReturned = true;
+
+ return new Iterator>() {
+ @Override
+ public boolean hasNext() {
+ while (state != RELEASED && state != TERMINATED) {
+ update();
+ }
+ return state == RELEASED;
+ }
+
+ @Override
+ public KeyVal next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ state = REQUIRES_NEXT_OP;
+ return entry;
+ }
+
+ @Override
+ public void remove() {
+ cursor.delete();
+ }
+ };
+ }
+
+ private void executeCursorOp(final CursorOp op) {
+ final boolean found;
+ switch (op) {
+ case FIRST:
+ found = cursor.first();
+ break;
+ case LAST:
+ found = cursor.last();
+ break;
+ case NEXT:
+ found = cursor.next();
+ break;
+ case PREV:
+ found = cursor.prev();
+ break;
+ case GET_START_KEY:
+ found = cursor.get(range.getStart(), MDB_SET_RANGE);
+ break;
+ case GET_START_KEY_BACKWARD:
+ found = cursor.get(range.getStart(), MDB_SET_RANGE) || cursor.last();
+ break;
+ default:
+ throw new IllegalStateException("Unknown cursor operation");
+ }
+ entry.setK(found ? cursor.key() : null);
+ entry.setV(found ? cursor.val() : null);
+ }
+
+ private void executeIteratorOp() {
+ final IteratorOp op =
+ range.getType().iteratorOp(range.getStart(), range.getStop(), entry.key(), rangeComparator);
+ switch (op) {
+ case CALL_NEXT_OP:
+ executeCursorOp(range.getType().nextOp());
+ state = REQUIRES_ITERATOR_OP;
+ break;
+ case TERMINATE:
+ state = TERMINATED;
+ break;
+ case RELEASE:
+ state = RELEASED;
+ break;
+ default:
+ throw new IllegalStateException("Unknown operation");
+ }
+ }
+
+ private void update() {
+ switch (state) {
+ case REQUIRES_INITIAL_OP:
+ executeCursorOp(range.getType().initialOp());
+ state = REQUIRES_ITERATOR_OP;
+ break;
+ case REQUIRES_NEXT_OP:
+ executeCursorOp(range.getType().nextOp());
+ state = REQUIRES_ITERATOR_OP;
+ break;
+ case REQUIRES_ITERATOR_OP:
+ executeIteratorOp();
+ break;
+ case TERMINATED:
+ break;
+ default:
+ throw new IllegalStateException("Unknown state");
+ }
}
/**
- * The value.
+ * Holder for a key and value pair.
+ *
+ * The same holder instance will always be returned for a given iterator. The returned keys and
+ * values may change or point to different memory locations following changes in the iterator,
+ * cursor or transaction.
*
- * @return value
+ * @param buffer type
*/
- public T val() {
- return v;
- }
+ public static final class KeyVal {
+
+ private T k;
+ private T v;
+
+ /**
+ * Explicitly-defined default constructor to avoid warnings.
+ */
+ public KeyVal() {
+ }
+
+ /**
+ * The key.
+ *
+ * @return key
+ */
+ public T key() {
+ return k;
+ }
+
+ /**
+ * The value.
+ *
+ * @return value
+ */
+ public T val() {
+ return v;
+ }
- void setK(final T key) {
- this.k = key;
+ void setK(final T key) {
+ this.k = key;
+ }
+
+ void setV(final T val) {
+ this.v = val;
+ }
}
- void setV(final T val) {
- this.v = val;
+ /**
+ * Represents the internal {@link CursorIterable} state.
+ */
+ enum State {
+ REQUIRES_INITIAL_OP,
+ REQUIRES_NEXT_OP,
+ REQUIRES_ITERATOR_OP,
+ RELEASED,
+ TERMINATED
}
- }
-
- /** Represents the internal {@link CursorIterable} state. */
- enum State {
- REQUIRES_INITIAL_OP,
- REQUIRES_NEXT_OP,
- REQUIRES_ITERATOR_OP,
- RELEASED,
- TERMINATED
- }
}
diff --git a/src/main/java/org/lmdbjava/Dbi.java b/src/main/java/org/lmdbjava/Dbi.java
index ad1bb5a7..c622462b 100644
--- a/src/main/java/org/lmdbjava/Dbi.java
+++ b/src/main/java/org/lmdbjava/Dbi.java
@@ -54,6 +54,7 @@ public final class Dbi {
private final Env env;
private final byte[] name;
private final Pointer ptr;
+ private final BufferProxy proxy;
Dbi(
final Env env,
@@ -69,16 +70,14 @@ public final class Dbi {
}
this.env = env;
this.name = name == null ? null : Arrays.copyOf(name, name.length);
- if (comparator == null) {
- this.comparator = proxy.getComparator(flags);
- } else {
- this.comparator = comparator;
- }
+ this.proxy = proxy;
+ this.comparator = comparator;
final int flagsMask = mask(true, flags);
final Pointer dbiPtr = allocateDirect(RUNTIME, ADDRESS);
checkRc(LIB.mdb_dbi_open(txn.pointer(), name, flagsMask, dbiPtr));
ptr = dbiPtr.getPointer(0);
if (nativeCb) {
+ requireNonNull(comparator, "comparator cannot be null if nativeCb is set");
this.ccb =
(keyA, keyB) -> {
final T compKeyA = proxy.allocate();
@@ -96,6 +95,10 @@ public final class Dbi {
}
}
+ Pointer pointer() {
+ return ptr;
+ }
+
/**
* Close the database handle (normally unnecessary; use with caution).
*
@@ -275,7 +278,7 @@ public CursorIterable iterate(final Txn txn, final KeyRange range) {
env.checkNotClosed();
txn.checkReady();
}
- return new CursorIterable<>(txn, this, range, comparator);
+ return new CursorIterable<>(txn, this, range, comparator, proxy);
}
/**
diff --git a/src/main/java/org/lmdbjava/DbiFlags.java b/src/main/java/org/lmdbjava/DbiFlags.java
index 123ec9fd..2f5eadf6 100644
--- a/src/main/java/org/lmdbjava/DbiFlags.java
+++ b/src/main/java/org/lmdbjava/DbiFlags.java
@@ -55,14 +55,6 @@ public enum DbiFlags implements MaskedFlag {
* #MDB_INTEGERKEY} keys.
*/
MDB_INTEGERDUP(0x20),
- /**
- * Compare the numeric keys in native byte order and as unsigned.
- *
- * This option is applied only to {@link java.nio.ByteBuffer}, {@link org.agrona.DirectBuffer}
- * and byte array keys. {@link io.netty.buffer.ByteBuf} keys are always compared in native byte
- * order and as unsigned.
- */
- MDB_UNSIGNEDKEY(0x30, false),
/**
* With {@link #MDB_DUPSORT}, use reverse string dups.
*
@@ -78,24 +70,13 @@ public enum DbiFlags implements MaskedFlag {
MDB_CREATE(0x4_0000);
private final int mask;
- private final boolean propagatedToLmdb;
-
- DbiFlags(final int mask, final boolean propagatedToLmdb) {
- this.mask = mask;
- this.propagatedToLmdb = propagatedToLmdb;
- }
DbiFlags(final int mask) {
- this(mask, true);
+ this.mask = mask;
}
@Override
public int getMask() {
return mask;
}
-
- @Override
- public boolean isPropagatedToLmdb() {
- return propagatedToLmdb;
- }
}
diff --git a/src/main/java/org/lmdbjava/DirectBufferProxy.java b/src/main/java/org/lmdbjava/DirectBufferProxy.java
index 156e60e9..5022ed02 100644
--- a/src/main/java/org/lmdbjava/DirectBufferProxy.java
+++ b/src/main/java/org/lmdbjava/DirectBufferProxy.java
@@ -111,12 +111,12 @@ protected DirectBuffer allocate() {
}
@Override
- protected Comparator getSignedComparator() {
+ public Comparator getSignedComparator() {
return signedComparator;
}
@Override
- protected Comparator getUnsignedComparator() {
+ public Comparator getUnsignedComparator() {
return unsignedComparator;
}
diff --git a/src/main/java/org/lmdbjava/Env.java b/src/main/java/org/lmdbjava/Env.java
index 3db16119..1543282c 100644
--- a/src/main/java/org/lmdbjava/Env.java
+++ b/src/main/java/org/lmdbjava/Env.java
@@ -256,10 +256,17 @@ public Dbi openDbi(final String name, final DbiFlags... flags) {
/**
* Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link
- * Comparator} that is not invoked from native code.
+ * Comparator} for use by {@link CursorIterable} when comparing start/stop keys.
+ *
+ * It is very important that the passed comparator behaves in the same way as the comparator
+ * LMDB uses for its insertion order (for the type of data that will be stored in the database),
+ * or you fully understand the implications of them behaving differently.
+ * LMDB's comparator is unsigned lexicographical, unless {@link DbiFlags#MDB_INTEGERKEY} is used.
+ *
*
* @param name name of the database (or null if no name is required)
- * @param comparator custom comparator callback (or null to use default)
+ * @param comparator custom comparator for cursor start/stop key comparisons. If null,
+ * LMDB's comparator will be used.
* @param flags to open the database with
* @return a database that is ready to use
*/
@@ -271,11 +278,15 @@ public Dbi openDbi(
/**
* Convenience method that opens a {@link Dbi} with a UTF-8 database name and associated {@link
- * Comparator} that may be invoked from native code if specified.
+ * Comparator}. The comparator will be used by {@link CursorIterable} when comparing start/stop keys
+ * as a minimum. If nativeCb is {@code true}, this comparator will also be called by LMDB to determine
+ * insertion/iteration order. Calling back to a java comparator may significantly impact performance.
*
* @param name name of the database (or null if no name is required)
- * @param comparator custom comparator callback (or null to use default)
- * @param nativeCb whether native code calls back to the Java comparator
+ * @param comparator custom comparator for cursor start/stop key comparisons and optionally for
+ * LMDB to call back to. If null,
+ * LMDB's comparator will be used.
+ * @param nativeCb whether LMDB native code calls back to the Java comparator
* @param flags to open the database with
* @return a database that is ready to use
*/
diff --git a/src/main/java/org/lmdbjava/Key.java b/src/main/java/org/lmdbjava/Key.java
new file mode 100644
index 00000000..7fd8bbe2
--- /dev/null
+++ b/src/main/java/org/lmdbjava/Key.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright © 2016-2025 The LmdbJava Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.lmdbjava;
+
+import static java.util.Objects.requireNonNull;
+import static org.lmdbjava.BufferProxy.MDB_VAL_STRUCT_SIZE;
+import static org.lmdbjava.Library.RUNTIME;
+
+import jnr.ffi.Pointer;
+import jnr.ffi.provider.MemoryManager;
+
+/**
+ * Represents off-heap memory holding a key only.
+ *
+ * @param buffer type
+ */
+final class Key implements AutoCloseable {
+
+ private static final MemoryManager MEM_MGR = RUNTIME.getMemoryManager();
+ private boolean closed;
+ private T k;
+ private final BufferProxy proxy;
+ private final Pointer ptrArray;
+ private final Pointer ptrKey;
+ private final long ptrKeyAddr;
+
+ Key(final BufferProxy proxy) {
+ requireNonNull(proxy);
+ this.proxy = proxy;
+ this.k = proxy.allocate();
+ ptrKey = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE, false);
+ ptrKeyAddr = ptrKey.address();
+ ptrArray = MEM_MGR.allocateTemporary(MDB_VAL_STRUCT_SIZE * 2, false);
+ }
+
+ @Override
+ public void close() {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ proxy.deallocate(k);
+ }
+
+ T key() {
+ return k;
+ }
+
+ void keyIn(final T key) {
+ proxy.in(key, ptrKey, ptrKeyAddr);
+ }
+
+ T keyOut() {
+ k = proxy.out(k, ptrKey, ptrKeyAddr);
+ return k;
+ }
+
+ Pointer pointerKey() {
+ return ptrKey;
+ }
+}
diff --git a/src/main/java/org/lmdbjava/KeyRangeType.java b/src/main/java/org/lmdbjava/KeyRangeType.java
index ad67286d..07123e9a 100644
--- a/src/main/java/org/lmdbjava/KeyRangeType.java
+++ b/src/main/java/org/lmdbjava/KeyRangeType.java
@@ -322,12 +322,12 @@ CursorOp initialOp() {
* @param start start buffer
* @param stop stop buffer
* @param buffer current key returned by LMDB (may be null)
- * @param c comparator (required)
+ * @param rangeComparator comparator (required)
* @return response to this key
*/
> IteratorOp iteratorOp(
- final T start, final T stop, final T buffer, final C c) {
- requireNonNull(c, "Comparator required");
+ final T start, final T stop, final T buffer, final RangeComparator rangeComparator) {
+ requireNonNull(rangeComparator, "Comparator required");
if (buffer == null) {
return TERMINATE;
}
@@ -337,55 +337,55 @@ > IteratorOp iteratorOp(
case FORWARD_AT_LEAST:
return RELEASE;
case FORWARD_AT_MOST:
- return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE;
case FORWARD_CLOSED:
- return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE;
case FORWARD_CLOSED_OPEN:
- return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE;
case FORWARD_GREATER_THAN:
- return c.compare(buffer, start) == 0 ? CALL_NEXT_OP : RELEASE;
+ return rangeComparator.compareToStartKey() == 0 ? CALL_NEXT_OP : RELEASE;
case FORWARD_LESS_THAN:
- return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE;
case FORWARD_OPEN:
- if (c.compare(buffer, start) == 0) {
+ if (rangeComparator.compareToStartKey() == 0) {
return CALL_NEXT_OP;
}
- return c.compare(buffer, stop) >= 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() >= 0 ? TERMINATE : RELEASE;
case FORWARD_OPEN_CLOSED:
- if (c.compare(buffer, start) == 0) {
+ if (rangeComparator.compareToStartKey() == 0) {
return CALL_NEXT_OP;
}
- return c.compare(buffer, stop) > 0 ? TERMINATE : RELEASE;
+ return rangeComparator.compareToStopKey() > 0 ? TERMINATE : RELEASE;
case BACKWARD_ALL:
return RELEASE;
case BACKWARD_AT_LEAST:
- return c.compare(buffer, start) > 0 ? CALL_NEXT_OP : RELEASE; // rewind
+ return rangeComparator.compareToStartKey() > 0 ? CALL_NEXT_OP : RELEASE; // rewind
case BACKWARD_AT_MOST:
- return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE;
case BACKWARD_CLOSED:
- if (c.compare(buffer, start) > 0) {
+ if (rangeComparator.compareToStartKey() > 0) {
return CALL_NEXT_OP; // rewind
}
- return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE;
case BACKWARD_CLOSED_OPEN:
- if (c.compare(buffer, start) > 0) {
+ if (rangeComparator.compareToStartKey() > 0) {
return CALL_NEXT_OP; // rewind
}
- return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE;
case BACKWARD_GREATER_THAN:
- return c.compare(buffer, start) >= 0 ? CALL_NEXT_OP : RELEASE;
+ return rangeComparator.compareToStartKey() >= 0 ? CALL_NEXT_OP : RELEASE;
case BACKWARD_LESS_THAN:
- return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE;
case BACKWARD_OPEN:
- if (c.compare(buffer, start) >= 0) {
+ if (rangeComparator.compareToStartKey() >= 0) {
return CALL_NEXT_OP; // rewind
}
- return c.compare(buffer, stop) > 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() > 0 ? RELEASE : TERMINATE;
case BACKWARD_OPEN_CLOSED:
- if (c.compare(buffer, start) >= 0) {
+ if (rangeComparator.compareToStartKey() >= 0) {
return CALL_NEXT_OP; // rewind
}
- return c.compare(buffer, stop) >= 0 ? RELEASE : TERMINATE;
+ return rangeComparator.compareToStopKey() >= 0 ? RELEASE : TERMINATE;
default:
throw new IllegalStateException("Invalid type");
}
diff --git a/src/main/java/org/lmdbjava/Library.java b/src/main/java/org/lmdbjava/Library.java
index ef9b9b35..6d8122d2 100644
--- a/src/main/java/org/lmdbjava/Library.java
+++ b/src/main/java/org/lmdbjava/Library.java
@@ -235,6 +235,8 @@ public interface Lmdb {
void mdb_txn_reset(@In Pointer txn);
+ int mdb_cmp(@In Pointer txn, @In Pointer dbi, @In Pointer key1, @In Pointer key2);
+
Pointer mdb_version(IntByReference major, IntByReference minor, IntByReference patch);
}
}
diff --git a/src/main/java/org/lmdbjava/RangeComparator.java b/src/main/java/org/lmdbjava/RangeComparator.java
new file mode 100644
index 00000000..162584b1
--- /dev/null
+++ b/src/main/java/org/lmdbjava/RangeComparator.java
@@ -0,0 +1,19 @@
+package org.lmdbjava;
+
+/**
+ * For comparing a cursor's current key against a {@link KeyRange}'s start/stop key.
+ */
+interface RangeComparator {
+
+ /**
+ * Compare the cursor's current key to the range start key. Equivalent to compareTo(currentKey,
+ * startKey)
+ */
+ int compareToStartKey();
+
+ /**
+ * Compare the cursor's current key to the range stop key. Equivalent to compareTo(currentKey,
+ * stopKey)
+ */
+ int compareToStopKey();
+}
diff --git a/src/test/java/org/lmdbjava/ComparatorTest.java b/src/test/java/org/lmdbjava/ComparatorTest.java
index 3e265cee..dad84ed2 100644
--- a/src/test/java/org/lmdbjava/ComparatorTest.java
+++ b/src/test/java/org/lmdbjava/ComparatorTest.java
@@ -135,7 +135,7 @@ private static final class ByteArrayRunner implements ComparatorRunner {
@Override
public int compare(final byte[] o1, final byte[] o2) {
- final Comparator c = PROXY_BA.getComparator();
+ final Comparator c = PROXY_BA.getUnsignedComparator();
return c.compare(o1, o2);
}
}
@@ -145,7 +145,7 @@ private static final class ByteBufferRunner implements ComparatorRunner {
@Override
public int compare(final byte[] o1, final byte[] o2) {
- final Comparator c = PROXY_OPTIMAL.getComparator();
+ final Comparator c = PROXY_OPTIMAL.getUnsignedComparator();
// Convert arrays to buffers that are larger than the array, with
// limit set at the array length. One buffer bigger than the other.
@@ -189,7 +189,7 @@ private static final class DirectBufferRunner implements ComparatorRunner {
public int compare(final byte[] o1, final byte[] o2) {
final DirectBuffer o1b = new UnsafeBuffer(o1);
final DirectBuffer o2b = new UnsafeBuffer(o2);
- final Comparator c = PROXY_DB.getComparator();
+ final Comparator c = PROXY_DB.getUnsignedComparator();
return c.compare(o1b, o2b);
}
}
@@ -223,7 +223,7 @@ public int compare(final byte[] o1, final byte[] o2) {
final ByteBuf o2b = DEFAULT.directBuffer(o2.length);
o1b.writeBytes(o1);
o2b.writeBytes(o2);
- final Comparator c = PROXY_NETTY.getComparator();
+ final Comparator c = PROXY_NETTY.getUnsignedComparator();
return c.compare(o1b, o2b);
}
}
diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
new file mode 100644
index 00000000..a9647d56
--- /dev/null
+++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
@@ -0,0 +1,163 @@
+package org.lmdbjava;
+
+import static com.jakewharton.byteunits.BinaryByteUnit.GIBIBYTES;
+import static org.lmdbjava.DbiFlags.MDB_CREATE;
+import static org.lmdbjava.Env.create;
+import static org.lmdbjava.EnvFlags.MDB_NOSUBDIR;
+import static org.lmdbjava.PutFlags.MDB_APPEND;
+import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE;
+import static org.lmdbjava.TestUtils.POSIX_MODE;
+import static org.lmdbjava.TestUtils.bb;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class CursorIterablePerfTest {
+
+ @Rule
+ public final TemporaryFolder tmp = new TemporaryFolder();
+
+// private static final int ITERATIONS = 5_000_000;
+ private static final int ITERATIONS = 100_000;
+// private static final int ITERATIONS = 10;
+
+ private Dbi dbJavaComparator;
+ private Dbi dbLmdbComparator;
+ private Dbi dbCallbackComparator;
+ private List> dbs = new ArrayList<>();
+ private Env env;
+ private List data = new ArrayList<>(ITERATIONS);
+
+ @Before
+ public void before() throws IOException {
+ final File path = tmp.newFile();
+ final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL;
+ env =
+ create(bufferProxy)
+ .setMapSize(GIBIBYTES.toBytes(1))
+ .setMaxReaders(1)
+ .setMaxDbs(3)
+ .open(path, POSIX_MODE, MDB_NOSUBDIR);
+
+ // Use a java comparator for start/stop keys only
+ dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE);
+ // Use LMDB comparator for start/stop keys
+ dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE);
+ // Use a java comparator for start/stop keys and as a callback comparator
+ dbCallbackComparator = env.openDbi(
+ "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE);
+
+ dbs.add(dbJavaComparator);
+ dbs.add(dbLmdbComparator);
+ dbs.add(dbCallbackComparator);
+
+ populateList();
+ }
+
+ private void populateList() {
+ for (int i = 0; i < ITERATIONS * 2; i+=2) {
+ data.add(i);
+ }
+ }
+
+ private void populateDatabases(final boolean randomOrder) {
+ System.out.println("Clear then populate databases");
+
+ final List data;
+ if (randomOrder) {
+ data = new ArrayList<>(this.data);
+ Collections.shuffle(data);
+ } else {
+ data = this.data;
+ }
+
+ for (int round = 0; round < 3; round++) {
+ System.out.println("round: " + round + " -----------------------------------------");
+
+ for (final Dbi db : dbs) {
+ // Clean out the db first
+ try (Txn txn = env.txnWrite();
+ final Cursor cursor = db.openCursor(txn)) {
+ while (cursor.next()) {
+ cursor.delete();
+ }
+ }
+
+ final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
+ final Instant start = Instant.now();
+ try (Txn txn = env.txnWrite()) {
+ for (final Integer i : data) {
+ if (randomOrder) {
+ db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE);
+ } else {
+ db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND);
+ }
+ }
+ txn.commit();
+ }
+ final Duration duration = Duration.between(start, Instant.now());
+ System.out.println("DB: " + dbName
+ + " - Loaded in duration: " + duration
+ + ", millis: " + duration.toMillis());
+ }
+ }
+ }
+
+ @After
+ public void after() {
+ env.close();
+ tmp.delete();
+ }
+
+ @Test
+ public void comparePerf_sequential() {
+ comparePerf(false);
+ }
+
+ @Test
+ public void comparePerf_random() {
+ comparePerf(true);
+ }
+
+ public void comparePerf(final boolean randomOrder) {
+ populateDatabases(randomOrder);
+ final ByteBuffer startKeyBuf = bb(data.getFirst());
+ final ByteBuffer stopKeyBuf = bb(data.getLast());
+ final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf);
+
+ System.out.println("\nIterating over all entries");
+ for (int round = 0; round < 3; round++) {
+ System.out.println("round: " + round + " -----------------------------------------");
+ for (final Dbi db : dbs) {
+ final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
+
+ final Instant start = Instant.now();
+ int cnt = 0;
+ // Exercise the stop key comparator on every entry
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn, keyRange)) {
+ for (final CursorIterable.KeyVal kv : c) {
+ cnt++;
+ }
+ }
+ final Duration duration = Duration.between(start, Instant.now());
+ System.out.println("DB: " + dbName
+ + " - Iterated in duration: " + duration
+ + ", millis: " + duration.toMillis()
+ + ", cnt: " + cnt);
+ }
+ }
+ }
+}
diff --git a/src/test/java/org/lmdbjava/CursorIterableTest.java b/src/test/java/org/lmdbjava/CursorIterableTest.java
index bd23bc55..96be0816 100644
--- a/src/test/java/org/lmdbjava/CursorIterableTest.java
+++ b/src/test/java/org/lmdbjava/CursorIterableTest.java
@@ -43,6 +43,8 @@
import static org.lmdbjava.KeyRange.openClosedBackward;
import static org.lmdbjava.PutFlags.MDB_NOOVERWRITE;
import static org.lmdbjava.TestUtils.DB_1;
+import static org.lmdbjava.TestUtils.DB_2;
+import static org.lmdbjava.TestUtils.DB_3;
import static org.lmdbjava.TestUtils.POSIX_MODE;
import static org.lmdbjava.TestUtils.bb;
@@ -69,7 +71,10 @@
public final class CursorIterableTest {
@Rule public final TemporaryFolder tmp = new TemporaryFolder();
- private Dbi db;
+ private Dbi dbJavaComparator;
+ private Dbi dbLmdbComparator;
+ private Dbi dbCallbackComparator;
+ private List> dbs = new ArrayList<>();
private Env env;
private Deque list;
@@ -116,19 +121,39 @@ public void atMostTest() {
@Before
public void before() throws IOException {
final File path = tmp.newFile();
+ final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL;
env =
- create()
+ create(bufferProxy)
.setMapSize(KIBIBYTES.toBytes(256))
.setMaxReaders(1)
- .setMaxDbs(1)
+ .setMaxDbs(3)
.open(path, POSIX_MODE, MDB_NOSUBDIR);
- db = env.openDbi(DB_1, MDB_CREATE);
- populateDatabase(db);
+
+ // Use a java comparator for start/stop keys only
+ dbJavaComparator = env.openDbi(DB_1, bufferProxy.getUnsignedComparator(), MDB_CREATE);
+ // Use LMDB comparator for start/stop keys
+ dbLmdbComparator = env.openDbi(DB_2, MDB_CREATE);
+ // Use a java comparator for start/stop keys and as a callback comparaotr
+ dbCallbackComparator = env.openDbi(
+ DB_3, bufferProxy.getUnsignedComparator(), true, MDB_CREATE);
+
+ populateList();
+
+ populateDatabase(dbJavaComparator);
+ populateDatabase(dbLmdbComparator);
+ populateDatabase(dbCallbackComparator);
+
+ dbs.add(dbJavaComparator);
+ dbs.add(dbLmdbComparator);
+ dbs.add(dbCallbackComparator);
}
- private void populateDatabase(final Dbi dbi) {
+ private void populateList() {
list = new LinkedList<>();
list.addAll(asList(2, 3, 4, 5, 6, 7, 8, 9));
+ }
+
+ private void populateDatabase(final Dbi dbi) {
try (Txn txn = env.txnWrite()) {
final Cursor c = dbi.openCursor(txn);
c.put(bb(2), bb(3), MDB_NOOVERWRITE);
@@ -166,6 +191,14 @@ public void closedTest() {
verify(closed(bb(1), bb(7)), 2, 4, 6);
}
+ public void closedTest1() {
+ verify(dbLmdbComparator, closed(bb(3), bb(7)), 4, 6);
+ }
+
+ public void closedTest2() {
+ verify(dbJavaComparator, closed(bb(3), bb(7)), 4, 6);
+ }
+
@Test
public void greaterThanBackwardTest() {
verify(greaterThanBackward(bb(6)), 4, 2);
@@ -181,30 +214,39 @@ public void greaterThanTest() {
@Test(expected = IllegalStateException.class)
public void iterableOnlyReturnedOnce() {
- try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn)) {
- c.iterator(); // ok
- c.iterator(); // fails
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn)) {
+ c.iterator(); // ok
+ c.iterator(); // fails
+ }
}
}
@Test
public void iterate() {
- try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn)) {
- for (final KeyVal kv : c) {
- assertThat(kv.key().getInt(), is(list.pollFirst()));
- assertThat(kv.val().getInt(), is(list.pollFirst()));
+ for (final Dbi db : dbs) {
+ populateList();
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn)) {
+
+ int cnt = 0;
+ for (final KeyVal kv : c) {
+ assertThat(kv.key().getInt(), is(list.pollFirst()));
+ assertThat(kv.val().getInt(), is(list.pollFirst()));
+ }
}
}
}
@Test(expected = IllegalStateException.class)
public void iteratorOnlyReturnedOnce() {
- try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn)) {
- c.iterator(); // ok
- c.iterator(); // fails
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn)) {
+ c.iterator(); // ok
+ c.iterator(); // fails
+ }
}
}
@@ -222,16 +264,19 @@ public void lessThanTest() {
@Test(expected = NoSuchElementException.class)
public void nextThrowsNoSuchElementExceptionIfNoMoreElements() {
- try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn)) {
- final Iterator> i = c.iterator();
- while (i.hasNext()) {
- final KeyVal kv = i.next();
- assertThat(kv.key().getInt(), is(list.pollFirst()));
- assertThat(kv.val().getInt(), is(list.pollFirst()));
+ for (final Dbi db : dbs) {
+ populateList();
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn)) {
+ final Iterator> i = c.iterator();
+ while (i.hasNext()) {
+ final KeyVal kv = i.next();
+ assertThat(kv.key().getInt(), is(list.pollFirst()));
+ assertThat(kv.val().getInt(), is(list.pollFirst()));
+ }
+ assertThat(i.hasNext(), is(false));
+ i.next();
}
- assertThat(i.hasNext(), is(false));
- i.next();
}
}
@@ -284,81 +329,148 @@ public void openTest() {
@Test
public void removeOddElements() {
- verify(all(), 2, 4, 6, 8);
- int idx = -1;
- try (Txn txn = env.txnWrite()) {
- try (CursorIterable ci = db.iterate(txn)) {
- final Iterator> c = ci.iterator();
- while (c.hasNext()) {
- c.next();
- idx++;
- if (idx % 2 == 0) {
- c.remove();
+ for (final Dbi db : dbs) {
+ verify(db, all(), 2, 4, 6, 8);
+ int idx = -1;
+ try (Txn txn = env.txnWrite()) {
+ try (CursorIterable ci = db.iterate(txn)) {
+ final Iterator> c = ci.iterator();
+ while (c.hasNext()) {
+ c.next();
+ idx++;
+ if (idx % 2 == 0) {
+ c.remove();
+ }
}
}
+ txn.commit();
}
- txn.commit();
+ verify(db, all(), 4, 8);
}
- verify(all(), 4, 8);
}
@Test(expected = Env.AlreadyClosedException.class)
public void nextWithClosedEnvTest() {
- try (Txn txn = env.txnRead()) {
- try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
- final Iterator> c = ci.iterator();
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnRead()) {
+ try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
+ final Iterator> c = ci.iterator();
- env.close();
- c.next();
+ env.close();
+ c.next();
+ }
}
}
}
@Test(expected = Env.AlreadyClosedException.class)
public void removeWithClosedEnvTest() {
- try (Txn txn = env.txnWrite()) {
- try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
- final Iterator> c = ci.iterator();
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnWrite()) {
+ try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
+ final Iterator> c = ci.iterator();
- final KeyVal keyVal = c.next();
- assertThat(keyVal, Matchers.notNullValue());
+ final KeyVal keyVal = c.next();
+ assertThat(keyVal, Matchers.notNullValue());
- env.close();
- c.remove();
+ env.close();
+ c.remove();
+ }
}
}
}
@Test(expected = Env.AlreadyClosedException.class)
public void hasNextWithClosedEnvTest() {
- try (Txn txn = env.txnRead()) {
- try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
- final Iterator> c = ci.iterator();
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnRead()) {
+ try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
+ final Iterator> c = ci.iterator();
- env.close();
- c.hasNext();
+ env.close();
+ c.hasNext();
+ }
}
}
}
@Test(expected = Env.AlreadyClosedException.class)
public void forEachRemainingWithClosedEnvTest() {
- try (Txn txn = env.txnRead()) {
- try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
- final Iterator> c = ci.iterator();
+ for (final Dbi db : dbs) {
+ try (Txn txn = env.txnRead()) {
+ try (CursorIterable ci = db.iterate(txn, KeyRange.all())) {
+ final Iterator> c = ci.iterator();
- env.close();
- c.forEachRemaining(keyVal -> {});
+ env.close();
+ c.forEachRemaining(keyVal -> {});
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testSignedVsUnsigned() {
+ final ByteBuffer val1 = bb(1);
+ final ByteBuffer val2 = bb(2);
+ final ByteBuffer val110 = bb(110);
+ final ByteBuffer val111 = bb(111);
+ final ByteBuffer val150 = bb(150);
+
+ final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL;
+ final Comparator unsignedComparator = bufferProxy.getUnsignedComparator();
+ final Comparator signedComparator = bufferProxy.getSignedComparator();
+
+ // Compare the same
+ assertThat(
+ unsignedComparator.compare(val1, val2),
+ Matchers.is(signedComparator.compare(val1, val2)));
+
+ // Compare differently
+ assertThat(
+ unsignedComparator.compare(val110, val150),
+ Matchers.not(signedComparator.compare(val110, val150)));
+
+ // Compare differently
+ assertThat(
+ unsignedComparator.compare(val111, val150),
+ Matchers.not(signedComparator.compare(val111, val150)));
+
+ // This will fail if the db is using a signed comparator for the start/stop keys
+ for (final Dbi db : dbs) {
+ db.put(val110, val110);
+ db.put(val150, val150);
+
+ final ByteBuffer startKeyBuf = val111;
+ KeyRange keyRange = KeyRange.atLeastBackward(startKeyBuf);
+
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn, keyRange)) {
+ for (final CursorIterable.KeyVal kv : c) {
+ final int key = kv.key().getInt();
+ final int val = kv.val().getInt();
+// System.out.println("key: " + key + " val: " + val);
+ assertThat(key, is(110));
+ break;
+ }
}
}
}
private void verify(final KeyRange range, final int... expected) {
- verify(range, db, expected);
+ // Verify using all comparator types
+ for (final Dbi db : dbs) {
+ verify(range, db, expected);
+ }
+ }
+
+ private void verify(
+ final Dbi dbi, final KeyRange range, final int... expected) {
+ verify(range, dbi, expected);
}
private void verify(
final KeyRange range, final Dbi dbi, final int... expected) {
+
final List results = new ArrayList<>();
try (Txn txn = env.txnRead();
diff --git a/src/test/java/org/lmdbjava/DbiTest.java b/src/test/java/org/lmdbjava/DbiTest.java
index 1fa80f6e..9c5cdb2e 100644
--- a/src/test/java/org/lmdbjava/DbiTest.java
+++ b/src/test/java/org/lmdbjava/DbiTest.java
@@ -111,7 +111,7 @@ public void close() {
public void customComparator() {
final Comparator reverseOrder =
(o1, o2) -> {
- final int lexical = PROXY_OPTIMAL.getComparator().compare(o1, o2);
+ final int lexical = PROXY_OPTIMAL.getUnsignedComparator().compare(o1, o2);
if (lexical == 0) {
return 0;
}
@@ -144,7 +144,7 @@ public void dbOpenMaxDatabases() {
@Test
public void dbiWithComparatorThreadSafety() {
final DbiFlags[] flags = new DbiFlags[] {MDB_CREATE, MDB_INTEGERKEY};
- final Comparator c = PROXY_OPTIMAL.getComparator(flags);
+ final Comparator c = PROXY_OPTIMAL.getUnsignedComparator();
final Dbi db = env.openDbi(DB_1, c, true, flags);
final List keys = range(0, 1_000).boxed().collect(toList());
diff --git a/src/test/java/org/lmdbjava/KeyRangeTest.java b/src/test/java/org/lmdbjava/KeyRangeTest.java
index 6e104bbf..0197bf11 100644
--- a/src/test/java/org/lmdbjava/KeyRangeTest.java
+++ b/src/test/java/org/lmdbjava/KeyRangeTest.java
@@ -195,7 +195,10 @@ private void verify(final KeyRange range, final int... expected) {
IteratorOp op;
do {
- op = range.getType().iteratorOp(range.getStart(), range.getStop(), buff, Integer::compare);
+ final Integer finalBuff = buff;
+ final RangeComparator rangeComparator =
+ CursorIterable.createJavaRangeComparator(range, Integer::compareTo, () -> finalBuff);
+ op = range.getType().iteratorOp(range.getStart(), range.getStop(), buff, rangeComparator);
switch (op) {
case CALL_NEXT_OP:
buff = cursor.apply(range.getType().nextOp(), range.getStart());
diff --git a/src/test/java/org/lmdbjava/TestUtils.java b/src/test/java/org/lmdbjava/TestUtils.java
index 42dcf052..f3d3974b 100644
--- a/src/test/java/org/lmdbjava/TestUtils.java
+++ b/src/test/java/org/lmdbjava/TestUtils.java
@@ -30,6 +30,8 @@
final class TestUtils {
public static final String DB_1 = "test-db-1";
+ public static final String DB_2 = "test-db-2";
+ public static final String DB_3 = "test-db-3";
public static final int POSIX_MODE = 0664;
From 46f8d08194d989529f87b447f48fad0214c10e80 Mon Sep 17 00:00:00 2001
From: at055612 <22818309+at055612@users.noreply.github.com>
Date: Thu, 6 Mar 2025 17:27:25 +0000
Subject: [PATCH 02/90] gh-249 Remove non-J8 features, use indent size 2
---
.../org/lmdbjava/CursorIterablePerfTest.java | 220 +++++++++---------
1 file changed, 110 insertions(+), 110 deletions(-)
diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
index a9647d56..ab94f85f 100644
--- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
+++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
@@ -29,135 +29,135 @@ public class CursorIterablePerfTest {
@Rule
public final TemporaryFolder tmp = new TemporaryFolder();
-// private static final int ITERATIONS = 5_000_000;
- private static final int ITERATIONS = 100_000;
-// private static final int ITERATIONS = 10;
-
- private Dbi dbJavaComparator;
- private Dbi dbLmdbComparator;
- private Dbi dbCallbackComparator;
- private List> dbs = new ArrayList<>();
- private Env env;
- private List data = new ArrayList<>(ITERATIONS);
-
- @Before
- public void before() throws IOException {
- final File path = tmp.newFile();
- final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL;
- env =
- create(bufferProxy)
- .setMapSize(GIBIBYTES.toBytes(1))
- .setMaxReaders(1)
- .setMaxDbs(3)
- .open(path, POSIX_MODE, MDB_NOSUBDIR);
-
- // Use a java comparator for start/stop keys only
+ // private static final int ITERATIONS = 5_000_000;
+ private static final int ITERATIONS = 100_000;
+ // private static final int ITERATIONS = 10;
+
+ private Dbi dbJavaComparator;
+ private Dbi dbLmdbComparator;
+ private Dbi dbCallbackComparator;
+ private List> dbs = new ArrayList<>();
+ private Env env;
+ private List data = new ArrayList<>(ITERATIONS);
+
+ @Before
+ public void before() throws IOException {
+ final File path = tmp.newFile();
+ final BufferProxy bufferProxy = ByteBufferProxy.PROXY_OPTIMAL;
+ env =
+ create(bufferProxy)
+ .setMapSize(GIBIBYTES.toBytes(1))
+ .setMaxReaders(1)
+ .setMaxDbs(3)
+ .open(path, POSIX_MODE, MDB_NOSUBDIR);
+
+ // Use a java comparator for start/stop keys only
dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE);
- // Use LMDB comparator for start/stop keys
- dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE);
- // Use a java comparator for start/stop keys and as a callback comparator
+ // Use LMDB comparator for start/stop keys
+ dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE);
+ // Use a java comparator for start/stop keys and as a callback comparator
dbCallbackComparator = env.openDbi(
"CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE);
- dbs.add(dbJavaComparator);
- dbs.add(dbLmdbComparator);
- dbs.add(dbCallbackComparator);
+ dbs.add(dbJavaComparator);
+ dbs.add(dbLmdbComparator);
+ dbs.add(dbCallbackComparator);
- populateList();
+ populateList();
+ }
+
+ private void populateList() {
+ for (int i = 0; i < ITERATIONS * 2; i += 2) {
+ data.add(i);
}
+ }
- private void populateList() {
- for (int i = 0; i < ITERATIONS * 2; i+=2) {
- data.add(i);
- }
+ private void populateDatabases(final boolean randomOrder) {
+ System.out.println("Clear then populate databases");
+
+ final List data;
+ if (randomOrder) {
+ data = new ArrayList<>(this.data);
+ Collections.shuffle(data);
+ } else {
+ data = this.data;
}
- private void populateDatabases(final boolean randomOrder) {
- System.out.println("Clear then populate databases");
+ for (int round = 0; round < 3; round++) {
+ System.out.println("round: " + round + " -----------------------------------------");
- final List data;
- if (randomOrder) {
- data = new ArrayList<>(this.data);
- Collections.shuffle(data);
- } else {
- data = this.data;
+ for (final Dbi db : dbs) {
+ // Clean out the db first
+ try (Txn txn = env.txnWrite();
+ final Cursor cursor = db.openCursor(txn)) {
+ while (cursor.next()) {
+ cursor.delete();
+ }
}
- for (int round = 0; round < 3; round++) {
- System.out.println("round: " + round + " -----------------------------------------");
-
- for (final Dbi db : dbs) {
- // Clean out the db first
- try (Txn txn = env.txnWrite();
- final Cursor cursor = db.openCursor(txn)) {
- while (cursor.next()) {
- cursor.delete();
- }
- }
-
- final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
- final Instant start = Instant.now();
- try (Txn txn = env.txnWrite()) {
- for (final Integer i : data) {
- if (randomOrder) {
- db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE);
- } else {
- db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND);
- }
- }
- txn.commit();
- }
- final Duration duration = Duration.between(start, Instant.now());
+ final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
+ final Instant start = Instant.now();
+ try (Txn txn = env.txnWrite()) {
+ for (final Integer i : data) {
+ if (randomOrder) {
+ db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE);
+ } else {
+ db.put(txn, bb(i), bb(i + 1), MDB_NOOVERWRITE, MDB_APPEND);
+ }
+ }
+ txn.commit();
+ }
+ final Duration duration = Duration.between(start, Instant.now());
System.out.println("DB: " + dbName
+ " - Loaded in duration: " + duration
+ ", millis: " + duration.toMillis());
- }
- }
- }
-
- @After
- public void after() {
- env.close();
- tmp.delete();
+ }
}
-
- @Test
- public void comparePerf_sequential() {
- comparePerf(false);
- }
-
- @Test
- public void comparePerf_random() {
- comparePerf(true);
- }
-
- public void comparePerf(final boolean randomOrder) {
- populateDatabases(randomOrder);
- final ByteBuffer startKeyBuf = bb(data.getFirst());
- final ByteBuffer stopKeyBuf = bb(data.getLast());
- final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf);
-
- System.out.println("\nIterating over all entries");
- for (int round = 0; round < 3; round++) {
- System.out.println("round: " + round + " -----------------------------------------");
- for (final Dbi db : dbs) {
- final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
-
- final Instant start = Instant.now();
- int cnt = 0;
- // Exercise the stop key comparator on every entry
- try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn, keyRange)) {
- for (final CursorIterable.KeyVal kv : c) {
- cnt++;
- }
- }
- final Duration duration = Duration.between(start, Instant.now());
+ }
+
+ @After
+ public void after() {
+ env.close();
+ tmp.delete();
+ }
+
+ @Test
+ public void comparePerf_sequential() {
+ comparePerf(false);
+ }
+
+ @Test
+ public void comparePerf_random() {
+ comparePerf(true);
+ }
+
+ public void comparePerf(final boolean randomOrder) {
+ populateDatabases(randomOrder);
+ final ByteBuffer startKeyBuf = bb(data.get(0));
+ final ByteBuffer stopKeyBuf = bb(data.get(data.size() - 1));
+ final KeyRange keyRange = KeyRange.closed(startKeyBuf, stopKeyBuf);
+
+ System.out.println("\nIterating over all entries");
+ for (int round = 0; round < 3; round++) {
+ System.out.println("round: " + round + " -----------------------------------------");
+ for (final Dbi db : dbs) {
+ final String dbName = new String(db.getName(), StandardCharsets.UTF_8);
+
+ final Instant start = Instant.now();
+ int cnt = 0;
+ // Exercise the stop key comparator on every entry
+ try (Txn txn = env.txnRead();
+ CursorIterable c = db.iterate(txn, keyRange)) {
+ for (final CursorIterable.KeyVal kv : c) {
+ cnt++;
+ }
+ }
+ final Duration duration = Duration.between(start, Instant.now());
System.out.println("DB: " + dbName
+ " - Iterated in duration: " + duration
+ ", millis: " + duration.toMillis()
+ ", cnt: " + cnt);
- }
- }
+ }
}
+ }
}
From f92012ecc079149b2414925e0a077a75f82ba043 Mon Sep 17 00:00:00 2001
From: at055612 <22818309+at055612@users.noreply.github.com>
Date: Thu, 6 Mar 2025 17:33:40 +0000
Subject: [PATCH 03/90] gh-249 Fix indents
---
.../org/lmdbjava/CursorIterablePerfTest.java | 28 +++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
diff --git a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
index ab94f85f..99667b1d 100644
--- a/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
+++ b/src/test/java/org/lmdbjava/CursorIterablePerfTest.java
@@ -26,8 +26,8 @@
public class CursorIterablePerfTest {
- @Rule
- public final TemporaryFolder tmp = new TemporaryFolder();
+ @Rule
+ public final TemporaryFolder tmp = new TemporaryFolder();
// private static final int ITERATIONS = 5_000_000;
private static final int ITERATIONS = 100_000;
@@ -52,12 +52,12 @@ public void before() throws IOException {
.open(path, POSIX_MODE, MDB_NOSUBDIR);
// Use a java comparator for start/stop keys only
- dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE);
+ dbJavaComparator = env.openDbi("JavaComparator", bufferProxy.getUnsignedComparator(), MDB_CREATE);
// Use LMDB comparator for start/stop keys
dbLmdbComparator = env.openDbi("LmdbComparator", MDB_CREATE);
// Use a java comparator for start/stop keys and as a callback comparator
- dbCallbackComparator = env.openDbi(
- "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE);
+ dbCallbackComparator = env.openDbi(
+ "CallBackComparator", bufferProxy.getUnsignedComparator(), true, MDB_CREATE);
dbs.add(dbJavaComparator);
dbs.add(dbLmdbComparator);
@@ -89,7 +89,7 @@ private void populateDatabases(final boolean randomOrder) {
for (final Dbi db : dbs) {
// Clean out the db first
try (Txn txn = env.txnWrite();
- final Cursor cursor = db.openCursor(txn)) {
+ final Cursor cursor = db.openCursor(txn)) {
while (cursor.next()) {
cursor.delete();
}
@@ -108,9 +108,9 @@ private void populateDatabases(final boolean randomOrder) {
txn.commit();
}
final Duration duration = Duration.between(start, Instant.now());
- System.out.println("DB: " + dbName
- + " - Loaded in duration: " + duration
- + ", millis: " + duration.toMillis());
+ System.out.println("DB: " + dbName
+ + " - Loaded in duration: " + duration
+ + ", millis: " + duration.toMillis());
}
}
}
@@ -147,16 +147,16 @@ public void comparePerf(final boolean randomOrder) {
int cnt = 0;
// Exercise the stop key comparator on every entry
try (Txn txn = env.txnRead();
- CursorIterable c = db.iterate(txn, keyRange)) {
+ CursorIterable c = db.iterate(txn, keyRange)) {
for (final CursorIterable.KeyVal kv : c) {
cnt++;
}
}
final Duration duration = Duration.between(start, Instant.now());
- System.out.println("DB: " + dbName
- + " - Iterated in duration: " + duration
- + ", millis: " + duration.toMillis()
- + ", cnt: " + cnt);
+ System.out.println("DB: " + dbName
+ + " - Iterated in duration: " + duration
+ + ", millis: " + duration.toMillis()
+ + ", cnt: " + cnt);
}
}
}
From e1756d633d7cbd1d23a33ffa0fcaaaa44c06aacd Mon Sep 17 00:00:00 2001
From: at055612 <22818309+at055612@users.noreply.github.com>
Date: Thu, 6 Mar 2025 19:02:46 +0000
Subject: [PATCH 04/90] gh-249 Tidy code and refactor RangeComparator impls
---
src/main/java/org/lmdbjava/BufferProxy.java | 8 +-
.../java/org/lmdbjava/CursorIterable.java | 544 ++++++++++--------
src/main/java/org/lmdbjava/Env.java | 23 +-
src/main/java/org/lmdbjava/Key.java | 4 +-
src/main/java/org/lmdbjava/KeyRangeType.java | 4 +-
.../java/org/lmdbjava/RangeComparator.java | 26 +-
.../org/lmdbjava/CursorIterablePerfTest.java | 37 +-
.../java/org/lmdbjava/CursorIterableTest.java | 18 +-
src/test/java/org/lmdbjava/KeyRangeTest.java | 4 +-
9 files changed, 376 insertions(+), 292 deletions(-)
diff --git a/src/main/java/org/lmdbjava/BufferProxy.java b/src/main/java/org/lmdbjava/BufferProxy.java
index 26d9db74..ab7ba3a4 100644
--- a/src/main/java/org/lmdbjava/BufferProxy.java
+++ b/src/main/java/org/lmdbjava/BufferProxy.java
@@ -69,11 +69,9 @@ protected BufferProxy() {}
/**
* Get a suitable default {@link Comparator} to compare numeric key values as signed.
*
- *
- * Note: LMDB's default comparator is unsigned so if this is used only for the {@link CursorIterable}
- * start/stop key comparisons then its behaviour will differ from the iteration order. Use
- * with caution.
- *
+ * Note: LMDB's default comparator is unsigned so if this is used only for the {@link
+ * CursorIterable} start/stop key comparisons then its behaviour will differ from the iteration
+ * order. Use with caution.
*
* @return a comparator that can be used (never null)
*/
diff --git a/src/main/java/org/lmdbjava/CursorIterable.java b/src/main/java/org/lmdbjava/CursorIterable.java
index 6b92a9cd..7c487bae 100644
--- a/src/main/java/org/lmdbjava/CursorIterable.java
+++ b/src/main/java/org/lmdbjava/CursorIterable.java
@@ -42,266 +42,352 @@
*/
public final class CursorIterable implements Iterable