From cf552d348b736bb8776979d84a8cb7cad753ef5b Mon Sep 17 00:00:00 2001 From: Christopher Jenkins Date: Tue, 14 Jan 2025 18:49:35 -0800 Subject: [PATCH] Fix selectResult readAt and deleteStale methods (#8) * added delete by keys added delete by read/write methods * - Added options on deleteState overloads - Bug: Added readAt to be set when being deserialized (as it has just been read) * fix merge --- gradle.properties | 5 +- gradle/libs.versions.toml | 2 +- .../com/mercury/sqkon/db/EntityQueries.kt | 4 +- .../com/mercury/sqkon/db/KeyValueStorage.kt | 40 +++++++-- .../kotlin/com/mercury/sqkon/db/ResultRow.kt | 4 +- .../com/mercury/sqkon/db/metadata.sq | 6 ++ .../sqkon/db/KeyValueStorageStaleTest.kt | 87 ++++++++++++++++++- 7 files changed, 130 insertions(+), 18 deletions(-) diff --git a/gradle.properties b/gradle.properties index ec3e111..2ca5394 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,10 +4,9 @@ org.gradle.caching=true org.gradle.configuration-cache=true org.gradle.daemon=true org.gradle.parallel=true - # Maven GROUP=com.mercury.sqkon -VERSION_NAME=1.0.0-alpha02 +VERSION_NAME=1.0.0-alpha04 POM_NAME=Sqkon POM_INCEPTION_YEAR=2024 POM_URL=https://github.com/MercuryTechnologies/sqkon/ @@ -19,11 +18,9 @@ POM_SCM_CONNECTION=scm:git:git://github.com/MercuryTechnologies/sqkon.git POM_SCM_DEV_CONNECTION=scm:git:ssh://git@github.com/MercuryTechnologies/sqkon.git POM_DEVELOPER_NAME=MercuryTechnologies POM_DEVELOPER_URL=https://github.com/MercuryTechnologies/ - #Kotlin kotlin.code.style=official kotlin.daemon.jvmargs=-Xmx4G - #Android android.useAndroidX=true android.nonTransitiveRClass=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f0e3cab..f372e57 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -4,7 +4,7 @@ androidx-runner = "1.6.2" kotlin = "2.1.0" agp = "8.8.0" kotlinx-coroutines = "1.10.1" -kotlinx-serialization = { require = "1.7.3" } +kotlinx-serialization = { require = "1.8.0" } kotlinx-datetime = "0.6.1" paging = "3.3.0-alpha02-0.5.1" sqlDelight = "2.0.2" diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt index 5f68ddc..d5f17b9 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/EntityQueries.kt @@ -206,13 +206,15 @@ class EntityQueries( parameters = 1, bindArgs = { bindString(entityName) } )) - when(entityKeys?.size) { + when (entityKeys?.size) { null, 0 -> {} + 1 -> add(SqlQuery( where = "entity_key = ?", parameters = 1, bindArgs = { bindString(entityKeys.first()) } )) + else -> add(SqlQuery( where = "entity_key IN (${entityKeys.joinToString(",") { "?" }})", parameters = entityKeys.size, diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt index b4d244d..2045466 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/KeyValueStorage.kt @@ -292,6 +292,7 @@ open class KeyValueStorage( entity.deserialize()?.let { v -> ResultRow(entity, v) } } } + .distinctUntilChanged() } /** @@ -402,17 +403,40 @@ open class KeyValueStorage( * hours. You would call this function with `Clock.System.now().minus(1.days)`. This is not the same as * [deleteExpired] which is based on the `expires_at` field. * + * @param writeInstant if set, will delete rows that have not been written to before this time. + * @param readInstant if set, will delete rows that have not been read before this time. + * * @see deleteExpired */ suspend fun deleteStale( - writeInstant: Instant = Clock.System.now(), - readInstant: Instant = Clock.System.now() + writeInstant: Instant? = Clock.System.now(), + readInstant: Instant? = Clock.System.now() ) = transaction { - metadataQueries.purgeStale( - entity_name = entityName, - writeInstant = writeInstant.toEpochMilliseconds(), - readInstant = readInstant.toEpochMilliseconds() - ) + when { + writeInstant != null && readInstant != null -> { + metadataQueries.purgeStale( + entity_name = entityName, + writeInstant = writeInstant.toEpochMilliseconds(), + readInstant = readInstant.toEpochMilliseconds() + ) + } + + writeInstant != null -> { + metadataQueries.purgeStaleWrite( + entity_name = entityName, + writeInstant = writeInstant.toEpochMilliseconds() + ) + } + + readInstant != null -> { + metadataQueries.purgeStaleRead( + entity_name = entityName, + readInstant = readInstant.toEpochMilliseconds() + ) + } + + else -> return@transaction + } updateWriteAt( currentCoroutineContext()[RequestHash.Key]?.hash ?: (writeInstant.hashCode() + readInstant.hashCode()) @@ -429,7 +453,7 @@ open class KeyValueStorage( * * @see deleteExpired */ - suspend fun deleteState(instant: Instant = Clock.System.now()) { + suspend fun deleteState(instant: Instant) { deleteStale(instant, instant) } diff --git a/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt b/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt index d65b727..d1052ec 100644 --- a/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt +++ b/library/src/commonMain/kotlin/com/mercury/sqkon/db/ResultRow.kt @@ -1,5 +1,6 @@ package com.mercury.sqkon.db +import kotlinx.datetime.Clock import kotlinx.datetime.Instant data class ResultRow( @@ -14,7 +15,8 @@ data class ResultRow( addedAt = Instant.fromEpochMilliseconds(entity.added_at), updatedAt = Instant.fromEpochMilliseconds(entity.updated_at), expiresAt = entity.expires_at?.let { Instant.fromEpochMilliseconds(it) }, - readAt = entity.read_at?.let { Instant.fromEpochMilliseconds(it) }, + readAt = Clock.System.now(), // By reading this value, we are marking it as read, we just + // update the db async writeAt = Instant.fromEpochMilliseconds(entity.write_at), value = value, ) diff --git a/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq index b67b769..254df85 100644 --- a/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq +++ b/library/src/commonMain/sqldelight/com/mercury/sqkon/db/metadata.sq @@ -38,3 +38,9 @@ DELETE FROM entity WHERE entity_name = :entity_name AND write_at < :writeInstant AND (read_at IS NULL OR read_at < :readInstant); + +purgeStaleWrite: +DELETE FROM entity WHERE entity_name = :entity_name AND write_at < :writeInstant; + +purgeStaleRead: +DELETE FROM entity WHERE entity_name = :entity_name AND read_at < :readInstant; diff --git a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt index 48ebc04..aa22057 100644 --- a/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt +++ b/library/src/commonTest/kotlin/com/mercury/sqkon/db/KeyValueStorageStaleTest.kt @@ -6,10 +6,11 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.flow.first import kotlinx.coroutines.test.runTest import kotlinx.datetime.Clock -import org.junit.After -import org.junit.Test import java.lang.Thread.sleep +import kotlin.test.AfterTest +import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertNotNull class KeyValueStorageStaleTest { @@ -21,7 +22,7 @@ class KeyValueStorageStaleTest { "test-object", entityQueries, metadataQueries, mainScope ) - @After + @AfterTest fun tearDown() { mainScope.cancel() } @@ -53,6 +54,38 @@ class KeyValueStorageStaleTest { assertEquals(0, actualAfterDelete.size) } + @Test + fun insertAll_staleWrite_purgeReadNotStale() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + sleep(1) + val now = Clock.System.now() + sleep(1) + testObjectStorage.selectAll().first() + // Clean up older than now + testObjectStorage.deleteStale(writeInstant = null, readInstant = now) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(expected.size, actualAfterDelete.size) + } + + @Test + fun insertAll_staleWrite_purgeStaleWrite() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + sleep(1) + val now = Clock.System.now() + sleep(1) + testObjectStorage.selectAll().first() + // Clean up older than now + testObjectStorage.deleteStale(writeInstant = now, readInstant = null) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(0, actualAfterDelete.size) + } + @Test fun insertAll_readInPast() = runTest { val expected = (0..10).map { TestObject() } @@ -70,6 +103,41 @@ class KeyValueStorageStaleTest { assertEquals(expected.size, actualAfterDelete.size) } + @Test + fun insertAll_readInPast_purgeStaleRead() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + testObjectStorage.selectAll().first() + sleep(1) + val now = Clock.System.now() + sleep(1) + // write again so read is in the past + testObjectStorage.updateAll(expected) + // Read in the past write is after now + testObjectStorage.deleteStale(writeInstant = null, readInstant = now) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(0, actualAfterDelete.size) + } + + @Test + fun insertAll_readInPast_purgeWriteNotStale() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + testObjectStorage.selectAll().first() + val now = Clock.System.now() + sleep(10) + // write again so read is in the past + testObjectStorage.updateAll(expected) + // Read in the past write is after now + testObjectStorage.deleteStale(writeInstant = now, readInstant = null) + val actualAfterDelete = testObjectStorage.selectAll().first() + assertEquals(expected.size, actualAfterDelete.size) + } + @Test fun insertAll_staleRead() = runTest { val expected = (0..10).map { TestObject() } @@ -86,4 +154,17 @@ class KeyValueStorageStaleTest { assertEquals(0, actualAfterDelete.size) } + @Test + fun selectResult_readWriteSet() = runTest { + val expected = (0..10).map { TestObject() } + .associateBy { it.id } + .toSortedMap() + testObjectStorage.insertAll(expected) + val actual = testObjectStorage.selectResult().first() + actual.forEach { result -> + assertNotNull(result.readAt) + assertNotNull(result.value) + } + } + }