diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java index a0ef94ce123..8d8f393c88d 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/ValueStore.java @@ -94,8 +94,6 @@ class ValueStore extends AbstractValueFactory { private final static Logger logger = LoggerFactory.getLogger(ValueStore.class); - private static final long VALUE_EVICTION_INTERVAL = 60000; // 60 seconds - private static final byte URI_VALUE = 0x0; // 00 private static final byte LITERAL_VALUE = 0x1; // 01 @@ -186,18 +184,20 @@ class ValueStore extends AbstractValueFactory { private final ConcurrentCleaner cleaner = new ConcurrentCleaner(); + private final long valueEvictionInterval; + ValueStore(File dir, LmdbStoreConfig config) throws IOException { this.dir = dir; this.forceSync = config.getForceSync(); this.autoGrow = config.getAutoGrow(); this.mapSize = config.getValueDBSize(); + this.valueEvictionInterval = config.getValueEvictionInterval(); open(); valueCache = new LmdbValue[config.getValueCacheSize()]; valueIDCache = new ConcurrentCache<>(config.getValueIDCacheSize()); namespaceCache = new ConcurrentCache<>(config.getNamespaceCacheSize()); namespaceIDCache = new ConcurrentCache<>(config.getNamespaceIDCacheSize()); - setNewRevision(); // read maximum id from store @@ -935,6 +935,10 @@ private static boolean isCommonVocabulary(IRI nv) { } public void gcIds(Collection ids, Collection nextIds) throws IOException { + if (!enableGC()) { + return; + } + if (!ids.isEmpty()) { // wrap into read txn as resizeMap expects an active surrounding read txn readTransaction(env, (stack1, txn1) -> { @@ -967,9 +971,9 @@ public void gcIds(Collection ids, Collection nextIds) throws IOExcep deleteValueToIdMappings(stack, writeTxn, finalIds, finalNextIds); - invalidateRevisionOnCommit = true; + invalidateRevisionOnCommit = enableGC(); if (nextValueEvictionTime < 0) { - nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL; + nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval; } return null; }); @@ -1180,7 +1184,7 @@ void endTransaction(boolean commit) throws IOException { unusedRevisionIds.add(revisionId); } if (nextValueEvictionTime < 0) { - nextValueEvictionTime = System.currentTimeMillis() + VALUE_EVICTION_INTERVAL; + nextValueEvictionTime = System.currentTimeMillis() + this.valueEvictionInterval; } }); setNewRevision(); @@ -1614,6 +1618,10 @@ public LmdbLiteral getLmdbLiteral(Literal l) { } } + private boolean enableGC() { + return this.valueEvictionInterval > 0; + } + public void forceEvictionOfValues() { nextValueEvictionTime = 0L; } diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java index 572008314e5..4c072e91317 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfig.java @@ -10,6 +10,8 @@ *******************************************************************************/ package org.eclipse.rdf4j.sail.lmdb.config; +import java.time.Duration; + import org.eclipse.rdf4j.model.Model; import org.eclipse.rdf4j.model.Resource; import org.eclipse.rdf4j.model.ValueFactory; @@ -71,6 +73,8 @@ public class LmdbStoreConfig extends BaseSailConfig { private boolean autoGrow = true; + private long valueEvictionInterval = Duration.ofSeconds(60).toMillis(); + /*--------------* * Constructors * *--------------*/ @@ -92,7 +96,6 @@ public LmdbStoreConfig(String tripleIndexes, boolean forceSync) { /*---------* * Methods * *---------*/ - public String getTripleIndexes() { return tripleIndexes; } @@ -178,6 +181,15 @@ public LmdbStoreConfig setAutoGrow(boolean autoGrow) { return this; } + public long getValueEvictionInterval() { + return valueEvictionInterval; + } + + public LmdbStoreConfig setValueEvictionInterval(long valueEvictionInterval) { + this.valueEvictionInterval = valueEvictionInterval; + return this; + } + @Override public Resource export(Model m) { Resource implNode = super.export(m); @@ -211,6 +223,9 @@ public Resource export(Model m) { if (!autoGrow) { m.add(implNode, LmdbStoreSchema.AUTO_GROW, vf.createLiteral(false)); } + if (valueEvictionInterval != Duration.ofSeconds(60).toMillis()) { + m.add(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, vf.createLiteral(valueEvictionInterval)); + } return implNode; } @@ -304,6 +319,17 @@ public void parse(Model m, Resource implNode) throws SailConfigException { "Boolean value required for " + LmdbStoreSchema.AUTO_GROW + " property, found " + lit); } }); + + Models.objectLiteral(m.getStatements(implNode, LmdbStoreSchema.VALUE_EVICTION_INTERVAL, null)) + .ifPresent(lit -> { + try { + setValueEvictionInterval(lit.longValue()); + } catch (NumberFormatException e) { + throw new SailConfigException( + "Long value required for " + LmdbStoreSchema.VALUE_EVICTION_INTERVAL + + " property, found " + lit); + } + }); } catch (ModelException e) { throw new SailConfigException(e.getMessage(), e); } diff --git a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java index 63c28cfa018..8a9c5acca8d 100644 --- a/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java +++ b/core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreSchema.java @@ -71,6 +71,11 @@ public class LmdbStoreSchema { */ public final static IRI AUTO_GROW; + /** + * http://rdf4j.org/config/sail/lmdb#valueEvictionInterval + */ + public final static IRI VALUE_EVICTION_INTERVAL; + static { ValueFactory factory = SimpleValueFactory.getInstance(); TRIPLE_INDEXES = factory.createIRI(NAMESPACE, "tripleIndexes"); @@ -82,5 +87,6 @@ public class LmdbStoreSchema { NAMESPACE_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceCacheSize"); NAMESPACE_ID_CACHE_SIZE = factory.createIRI(NAMESPACE, "namespaceIDCacheSize"); AUTO_GROW = factory.createIRI(NAMESPACE, "autoGrow"); + VALUE_EVICTION_INTERVAL = factory.createIRI(NAMESPACE, "valueEvictionInterval"); } } diff --git a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java index fff2bf17b11..8850b889c5b 100644 --- a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java +++ b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/ValueStoreTest.java @@ -221,6 +221,54 @@ public void testGcURIs() throws Exception { } } + @Test + public void testDisableGc() throws Exception { + final Random random = new Random(1337); + final LmdbValue values[] = new LmdbValue[1000]; + + final ValueStore valueStore = new ValueStore( + new File(dataDir, "values"), new LmdbStoreConfig().setValueEvictionInterval(-1)); + + valueStore.startTransaction(true); + for (int i = 0; i < values.length; i++) { + values[i] = valueStore.createLiteral("This is a random literal:" + random.nextLong()); + valueStore.storeValue(values[i]); + } + valueStore.commit(); + + final ValueStoreRevision revBefore = valueStore.getRevision(); + + valueStore.startTransaction(true); + Set ids = new HashSet<>(); + for (int i = 0; i < 30; i++) { + ids.add(values[i].getInternalID()); + } + valueStore.gcIds(ids, new HashSet<>()); + valueStore.commit(); + + final ValueStoreRevision revAfter = valueStore.getRevision(); + + assertEquals("revisions must NOT change since GC is disabled", revBefore, revAfter); + + Arrays.fill(values, null); + valueStore.unusedRevisionIds.add(revBefore.getRevisionId()); + + valueStore.forceEvictionOfValues(); + valueStore.startTransaction(true); + valueStore.commit(); + + valueStore.startTransaction(true); + for (int i = 0; i < 30; i++) { + LmdbValue value = valueStore.createLiteral("This is a random literal:" + random.nextLong()); + values[i] = value; + valueStore.storeValue(value); + ids.remove(value.getInternalID()); + } + valueStore.commit(); + + assertNotEquals("IDs should NOT have been reused since GC is disabled", Collections.emptySet(), ids); + } + @AfterEach public void after() throws Exception { valueStore.close(); diff --git a/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfigTest.java b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfigTest.java new file mode 100644 index 00000000000..b3d7f41176e --- /dev/null +++ b/core/sail/lmdb/src/test/java/org/eclipse/rdf4j/sail/lmdb/config/LmdbStoreConfigTest.java @@ -0,0 +1,103 @@ +/******************************************************************************* + * Copyright (c) 2021 Eclipse RDF4J contributors. + * + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Distribution License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + *******************************************************************************/ +package org.eclipse.rdf4j.sail.lmdb.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.rdf4j.model.util.Values.bnode; +import static org.eclipse.rdf4j.sail.lmdb.config.LmdbStoreConfig.VALUE_CACHE_SIZE; + +import org.eclipse.rdf4j.model.BNode; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Literal; +import org.eclipse.rdf4j.model.Model; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.impl.LinkedHashModel; +import org.eclipse.rdf4j.model.util.ModelBuilder; +import org.eclipse.rdf4j.model.util.Values; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class LmdbStoreConfigTest { + + @ParameterizedTest + @ValueSource(longs = { 1, 205454, 0, -1231 }) + void testThatLmdbStoreConfigParseAndExportValueEvictionInterval(final long valueEvictionInterval) { + testParseAndExport( + LmdbStoreSchema.VALUE_EVICTION_INTERVAL, + Values.literal(valueEvictionInterval), + LmdbStoreConfig::getValueEvictionInterval, + valueEvictionInterval, + true + ); + } + + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void testThatLmdbStoreConfigParseAndExportAutoGrow(final boolean autoGrow) { + testParseAndExport( + LmdbStoreSchema.AUTO_GROW, + Values.literal(autoGrow), + LmdbStoreConfig::getAutoGrow, + autoGrow, + !autoGrow + ); + } + + @ParameterizedTest + @ValueSource(ints = { 1, 205454, 0, -1231 }) + void testThatLmdbStoreConfigParseAndExportValueCacheSize(final int valueCacheSize) { + testParseAndExport( + LmdbStoreSchema.VALUE_CACHE_SIZE, + Values.literal(valueCacheSize >= 0 ? valueCacheSize : VALUE_CACHE_SIZE), + LmdbStoreConfig::getValueCacheSize, + valueCacheSize >= 0 ? valueCacheSize : VALUE_CACHE_SIZE, + true + ); + } + + // TODO: Add more tests for other properties + + /** + * Generic method to test parsing and exporting of config properties. + * + * @param property The schema property to test + * @param value The literal value to use in the test + * @param getter Function to get the value from the config object + * @param expectedValue The expected value after parsing + * @param expectedContains The expected result of the contains check + * @param The type of the value being tested + */ + private void testParseAndExport( + IRI property, + Literal value, + java.util.function.Function getter, + T expectedValue, + boolean expectedContains + ) { + final BNode implNode = bnode(); + final LmdbStoreConfig lmdbStoreConfig = new LmdbStoreConfig(); + final Model configModel = new ModelBuilder() + .add(implNode, property, value) + .build(); + + // Parse the config + lmdbStoreConfig.parse(configModel, implNode); + assertThat(getter.apply(lmdbStoreConfig)).isEqualTo(expectedValue); + + // Export the config + final Model exportedModel = new LinkedHashModel(); + final Resource exportImplNode = lmdbStoreConfig.export(exportedModel); + + // Verify the export + assertThat(exportedModel.contains(exportImplNode, property, value)) + .isEqualTo(expectedContains); + } +} \ No newline at end of file