Skip to content

Commit

Permalink
fix(model-server): use one large transaction when putting many values
Browse files Browse the repository at this point in the history
So far, when putting multiple values via objects/put, a database
transaction was created and committed for every value. That is
unbelievably slow with a real database. This commit changes the
implementation of the endpoint to use `putAll` instead of multiple `put`
calls, effectively reducing the whole operation to a single transaction.

This is a `fix` commit and not a `perf` one as the previous
implementation completely broke some known use cases due to ending in
timeouts.
  • Loading branch information
languitar committed Jan 3, 2024
1 parent 3f42ecd commit 1b00161
Showing 1 changed file with 13 additions and 13 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -225,26 +225,26 @@ class ModelReplicationServer(val repositoriesManager: RepositoriesManager) {
}
route("objects") {
put {
var writtenEntries = 0
withContext(Dispatchers.IO) {
var isKey = true
var key = ""
call.receiveStream().bufferedReader().lineSequence().forEach { line ->
if (isKey) {
key = line
} else {
val value = line
val writtenEntries = withContext(Dispatchers.IO) {
val entries = call.receiveStream().bufferedReader().use { reader ->
reader.lineSequence().windowed(2, 2).map {
val key = it[0]
val value = it[1]

require(HashUtil.isSha256(key)) {
"This API cannot be used to store other entries than serialized objects." +
" The key is expected to be a SHA256 hash over the value: $key -> $value"
}
val expectedKey = HashUtil.sha256(value)
require(expectedKey == key) { "Hash mismatch. Expected $expectedKey, but $key was provided. Value: $value" }
storeClient.put(key, value, true)
writtenEntries++
}
isKey = !isKey

key to value
}.toMap()
}

storeClient.putAll(entries, true)

entries.size
}
call.respondText("$writtenEntries objects received")
}
Expand Down

0 comments on commit 1b00161

Please sign in to comment.