GH-5260 LMDB Store: Add valueEvictionInterval config

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbSailStore.java

@@ -77,6 +77,8 @@ class LmdbSailStore implements SailStore {
 	private PersistentSetFactory<Long> setFactory;
 	private PersistentSet<Long> unusedIds, nextUnusedIds;
 
+	private final boolean enableGC;
+
 	/**
 	 * A fast non-blocking circular buffer backed by an array.
 	 *
@@ -188,6 +190,7 @@ public LmdbSailStore(File dataDir, LmdbStoreConfig config) throws IOException, S
 		this.unusedIds = setFactory.createSet("unusedIds", encode, decode);
 		this.nextUnusedIds = setFactory.createSet("nextUnusedIds", encode, decode);
 		boolean initialized = false;
+		this.enableGC = config.getValueEvictionInterval() > 0;
 		try {
 			namespaceStore = new NamespaceStore(dataDir);
 			valueStore = new ValueStore(new File(dataDir, "values"), config);
@@ -288,12 +291,12 @@ public EvaluationStatistics getEvaluationStatistics() {
 
 	@Override
 	public SailSource getExplicitSailSource() {
-		return new LmdbSailSource(true);
+		return new LmdbSailSource(true, this.enableGC);
 	}
 
 	@Override
 	public SailSource getInferredSailSource() {
-		return new LmdbSailSource(false);
+		return new LmdbSailSource(false, this.enableGC);
 	}
 
 	CloseableIteration<Resource> getContexts() throws IOException {
@@ -406,9 +409,11 @@ CloseableIteration<? extends Statement> createStatementIterator(
 	private final class LmdbSailSource extends BackingSailSource {
 
 		private final boolean explicit;
+		private final boolean enableGC;
 
-		public LmdbSailSource(boolean explicit) {
+		public LmdbSailSource(boolean explicit, boolean enableGC) {
 			this.explicit = explicit;
+			this.enableGC = enableGC;
 		}
 
 		@Override
@@ -418,7 +423,7 @@ public SailSource fork() {
 
 		@Override
 		public SailSink sink(IsolationLevel level) throws SailException {
-			return new LmdbSailSink(explicit);
+			return new LmdbSailSink(explicit, enableGC);
 		}
 
 		@Override
@@ -431,9 +436,11 @@ public LmdbSailDataset dataset(IsolationLevel level) throws SailException {
 	private final class LmdbSailSink implements SailSink {
 
 		private final boolean explicit;
+		private final boolean enableGC;
 
-		public LmdbSailSink(boolean explicit) throws SailException {
+		public LmdbSailSink(boolean explicit, boolean enableGC) throws SailException {
 			this.explicit = explicit;
+			this.enableGC = enableGC;
 		}
 
 		@Override
@@ -500,7 +507,11 @@ public void flush() throws SailException {
 							tripleStore.commit();
 							filterUsedIdsInTripleStore();
 						}
-						handleRemovedIdsInValueStore();
+						if (this.enableGC) {
+							handleRemovedIdsInValueStore();
+						} else {
+							logger.info("GC is disabled, unused ids will not be removed from the value store");
+						}
 						valueStore.commit();
 						// do not set flag to false until _after_ commit is successfully completed.
 						storeTxnStarted.set(false);

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java

@@ -94,8 +94,6 @@ class ValueStore extends AbstractValueFactory {
 
 	private final static Logger logger = LoggerFactory.getLogger(ValueStore.class);
 
-	private static final long VALUE_EVICTION_INTERVAL = 60000; // 60 seconds
-
 	private static final byte URI_VALUE = 0x0; // 00
 
 	private static final byte LITERAL_VALUE = 0x1; // 01
@@ -157,6 +155,8 @@ class ValueStore extends AbstractValueFactory {
 	private final boolean forceSync;
 	private final boolean autoGrow;
 	private boolean invalidateRevisionOnCommit = false;
+	private final long valueEvictionInterval;
+
 	/**
 	 * This lock is required to block transactions while auto-growing the map size.
 	 */
@@ -191,6 +191,7 @@ class ValueStore extends AbstractValueFactory {
 		this.forceSync = config.getForceSync();
 		this.autoGrow = config.getAutoGrow();
 		this.mapSize = config.getValueDBSize();
+		this.valueEvictionInterval = config.getValueEvictionInterval();
 		open();
 
 		valueCache = new LmdbValue[config.getValueCacheSize()];
@@ -969,7 +970,7 @@ public void gcIds(Collection<Long> ids, Collection<Long> nextIds) throws IOExcep
 
 					invalidateRevisionOnCommit = true;
 					if (nextValueEvictionTime < 0) {
-						nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL;
+						nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval;
 					}
 					return null;
 				});
@@ -1180,7 +1181,7 @@ void endTransaction(boolean commit) throws IOException {
 								unusedRevisionIds.add(revisionId);
 							}
 							if (nextValueEvictionTime < 0) {
-								nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL;
+								nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval;
 							}
 						});
 						setNewRevision();

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java

@@ -10,6 +10,8 @@
  *******************************************************************************/
 package org.eclipse.rdf4j.sail.lmdb.config;
 
+import java.time.Duration;
+
 import org.eclipse.rdf4j.model.Model;
 import org.eclipse.rdf4j.model.Resource;
 import org.eclipse.rdf4j.model.ValueFactory;
@@ -71,6 +73,8 @@ public class LmdbStoreConfig extends BaseSailConfig {
 
 	private boolean autoGrow = true;
 
+	private long valueEvictionInterval = Duration.ofSeconds(60).toMillis();
+
 	/*--------------*
 	 * Constructors *
 	 *--------------*/
@@ -93,6 +97,15 @@ public LmdbStoreConfig(String tripleIndexes, boolean forceSync) {
 	 * Methods *
 	 *---------*/
 
+	public long getValueEvictionInterval() {
+		return valueEvictionInterval;
+	}
+
+	public LmdbStoreConfig setValueEvictionInterval(long valueEvictionInterval) {
+		this.valueEvictionInterval = valueEvictionInterval;
+		return this;
+	}
+
 	public String getTripleIndexes() {
 		return tripleIndexes;
 	}
@@ -211,6 +224,9 @@ public Resource export(Model m) {
 		if (!autoGrow) {
 			m.add(implNode, LmdbStoreSchema.AUTO_GROW, vf.createLiteral(false));
 		}
+		if (valueEvictionInterval != Duration.ofSeconds(60).toMillis()) {
+			m.add(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, vf.createLiteral(valueEvictionInterval));
+		}
 		return implNode;
 	}
 
@@ -304,6 +320,17 @@ public void parse(Model m, Resource implNode) throws SailConfigException {
 							"Boolean value required for " + LmdbStoreSchema.AUTO_GROW + " property, found " + lit);
 				}
 			});
+
+			Models.objectLiteral(m.getStatements(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, null))
+					.ifPresent(lit -> {
+						try {
+							setValueEvictionInterval(lit.longValue());
+						} catch (NumberFormatException e) {
+							throw new SailConfigException(
+									"Long value required for " + LmdbStoreSchema.VALUE_EVICTION_INTERVAL
+											+ " property, found " + lit);
+						}
+					});
 		} catch (ModelException e) {
 			throw new SailConfigException(e.getMessage(), e);
 		}

core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java

@@ -71,6 +71,11 @@ public class LmdbStoreSchema {
 	 */
 	public final static IRI AUTO_GROW;
 
+	/**
+	 * <tt>http://rdf4j.org/config/sail/lmdb#valueEvictionInterval</tt>
+	 */
+	public final static IRI VALUE_EVICTION_INTERVAL;
+
 	static {
 		ValueFactory factory = SimpleValueFactory.getInstance();
 		TRIPLE_INDEXES = factory.createIRI(NAMESPACE, "tripleIndexes");
@@ -82,5 +87,6 @@ public class LmdbStoreSchema {
 		NAMESPACE_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceCacheSize");
 		NAMESPACE_ID_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceIDCacheSize");
 		AUTO_GROW = factory.createIRI(NAMESPACE, "autoGrow");
+		VALUE_EVICTION_INTERVAL = factory.createIRI(NAMESPACE, "valueEvictionInterval");
 	}
 }

core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfigTest.java

@@ -0,0 +1,80 @@
+package org.eclipse.rdf4j.sail.lmdb.config;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.eclipse.rdf4j.model.util.Values.bnode;
+
+import org.eclipse.rdf4j.model.BNode;
+import org.eclipse.rdf4j.model.IRI;
+import org.eclipse.rdf4j.model.Literal;
+import org.eclipse.rdf4j.model.Model;
+import org.eclipse.rdf4j.model.Resource;
+import org.eclipse.rdf4j.model.impl.LinkedHashModel;
+import org.eclipse.rdf4j.model.util.ModelBuilder;
+import org.eclipse.rdf4j.model.util.Values;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+class LmdbStoreConfigTest {
+
+	@ParameterizedTest
+	@ValueSource(longs = { 1, 205454, 0, -1231 })
+	void testThatLmdbStoreConfigParseAndExportValueEvictionInterval(final long valueEvictionInterval) {
+		testParseAndExport(
+				LmdbStoreSchema.VALUE_EVICTION_INTERVAL,
+				Values.literal(valueEvictionInterval),
+				LmdbStoreConfig::getValueEvictionInterval,
+				valueEvictionInterval,
+				true
+		);
+	}
+
+	@ParameterizedTest
+	@ValueSource(booleans = { true, false })
+	void testThatLmdbStoreConfigParseAndExportAutoGrow(final boolean autoGrow) {
+		testParseAndExport(
+				LmdbStoreSchema.AUTO_GROW,
+				Values.literal(autoGrow),
+				LmdbStoreConfig::getAutoGrow,
+				autoGrow,
+				!autoGrow
+		);
+	}
+
+	//TODO: Add more tests for other properties
+
+	/**
+	 * Generic method to test parsing and exporting of config properties.
+	 *
+	 * @param property         The schema property to test
+	 * @param value            The literal value to use in the test
+	 * @param getter           Function to get the value from the config object
+	 * @param expectedValue    The expected value after parsing
+	 * @param expectedContains The expected result of the contains check
+	 * @param <T>              The type of the value being tested
+	 */
+	private <T> void testParseAndExport(
+			IRI property,
+			Literal value,
+			java.util.function.Function<LmdbStoreConfig, T> getter,
+			T expectedValue,
+			boolean expectedContains
+	) {
+		final BNode implNode = bnode();
+		final LmdbStoreConfig lmdbStoreConfig = new LmdbStoreConfig();
+		final Model configModel = new ModelBuilder()
+				.add(implNode, property, value)
+				.build();
+
+		// Parse the config
+		lmdbStoreConfig.parse(configModel, implNode);
+		assertThat(getter.apply(lmdbStoreConfig)).isEqualTo(expectedValue);
+
+		// Export the config
+		final Model exportedModel = new LinkedHashModel();
+		final Resource exportImplNode = lmdbStoreConfig.export(exportedModel);
+
+		// Verify the export
+		assertThat(exportedModel.contains(exportImplNode, property, value))
+				.isEqualTo(expectedContains);
+	}
+}