diff --git a/packages/sync-service/lib/electric/shape_cache/file_storage/compaction.ex b/packages/sync-service/lib/electric/shape_cache/file_storage/compaction.ex index d223ec85b7..2267118649 100644 --- a/packages/sync-service/lib/electric/shape_cache/file_storage/compaction.ex +++ b/packages/sync-service/lib/electric/shape_cache/file_storage/compaction.ex @@ -6,6 +6,23 @@ defmodule Electric.ShapeCache.FileStorage.Compaction do alias Electric.ShapeCache.FileStorage.KeyIndex alias Electric.ShapeCache.FileStorage.ActionFile + # Compaction and race conditions + # + # `FileStorage` has a pointer to the last compacted offset (and the compacted log file name) + # which is updated atomically once the compaction is complete, so while it's ongoing, the + # pointer is not updated. + # + # While the log is compacted in place, it's actually a merged copy that's being + # compacted, not the original log. Original log is deleted after the compaction + # is complete and the pointer is updated. + # + # Any concurrent reads of the log that's being replaced are also OK: the `File.rename` + # on linux doesn't close the original file descriptor, so the reader will still see + # the original file, and we don't reuse file names. Any readers mid-file of the + # log that's being replaced but that read the chunk will continue from a correct chunk + # of the new file due to offset ordering being preserved. They might observe some updates + # more than once in a compacted form. + @spec compact_in_place({String.t(), String.t(), String.t()}, non_neg_integer(), (any(), any() -> any())) :: {String.t(), String.t(), String.t()}