diff --git a/src/Corax/Indexing/IndexWriter.cs b/src/Corax/Indexing/IndexWriter.cs index 8186b6317fbf..4b4fd2f91483 100644 --- a/src/Corax/Indexing/IndexWriter.cs +++ b/src/Corax/Indexing/IndexWriter.cs @@ -430,8 +430,12 @@ private void ProcessDeletes() Page lastVisitedPage = default; var fieldsByRootPage = GetIndexedFieldByRootPage(_fieldsTree); + var nullTermsMarkers = new HashSet(); - Querying.IndexSearcher.LoadNullTermMarkers(_nullEntriesPostingListsTree, nullTermsMarkers); + Querying.IndexSearcher.LoadSpecialTermMarkers(_nullEntriesPostingListsTree, nullTermsMarkers); + + var nonExistingTermsMarkers = new HashSet(); + Querying.IndexSearcher.LoadSpecialTermMarkers(_nonExistingEntriesPostingListsTree, nonExistingTermsMarkers); long dicId = CompactTree.GetDictionaryId(llt); @@ -447,22 +451,21 @@ private void ProcessDeletes() var termsPerEntryIndex = InsertTermsPerEntry(entryToDelete); - RecordTermDeletionsForEntry(entryTerms, llt, fieldsByRootPage, nullTermsMarkers, dicId, entryToDelete, termsPerEntryIndex); + RecordTermDeletionsForEntry(entryTerms, llt, fieldsByRootPage, nullTermsMarkers, nonExistingTermsMarkers, dicId, entryToDelete, termsPerEntryIndex); Container.Delete(llt, _entriesTermsContainerId, entryTermsId); } } - private void RecordTermDeletionsForEntry(Container.Item entryTerms, LowLevelTransaction llt, Dictionary fieldsByRootPage, HashSet nullTermMarkers, long dicId, long entryToDelete, int termsPerEntryIndex) + private void RecordTermDeletionsForEntry(Container.Item entryTerms, LowLevelTransaction llt, Dictionary fieldsByRootPage, HashSet nullTermMarkers, HashSet nonExistingTermMarkers, long dicId, long entryToDelete, int termsPerEntryIndex) { - var reader = new EntryTermsReader(llt, nullTermMarkers, entryTerms.Address, entryTerms.Length, dicId); + var reader = new EntryTermsReader(llt, nullTermMarkers, nonExistingTermMarkers, entryTerms.Address, entryTerms.Length, dicId); reader.Reset(); while (reader.MoveNextStoredField()) { - //Null/empty is not stored in container, just exists as marker. + // Null/empty is not stored in container, just exists as marker. if (reader.TermId == -1) continue; - Container.Delete(llt, _storedFieldsContainerId, reader.TermId); } reader.Reset(); @@ -475,15 +478,13 @@ private void RecordTermDeletionsForEntry(Container.Item entryTerms, LowLevelTran if (reader.IsNull) { - ref var nullTermLocation = ref CollectionsMarshal.GetValueRefOrAddDefault(field.Textual, Constants.NullValueSlice, out var nullExists); - if (nullExists == false) - { - nullTermLocation = field.Storage.Count; - field.Storage.AddByRef(new EntriesModifications(1)); - // We dont want to reclaim the term name - } - ref var nullTerm = ref field.Storage.GetAsRef(nullTermLocation); - nullTerm.Removal(_entriesAllocator, entryToDelete, termsPerEntryIndex, reader.Frequency); + RemoveSpecialTerm(field, reader, Constants.NullValueSlice, entryToDelete, termsPerEntryIndex); + continue; + } + + if (reader.IsNonExisting) + { + RemoveSpecialTerm(field, reader, Constants.NonExistingValueSlice, entryToDelete, termsPerEntryIndex); continue; } @@ -528,6 +529,19 @@ private void RecordTermDeletionsForEntry(Container.Item entryTerms, LowLevelTran term.Removal(_entriesAllocator, entryToDelete, termsPerEntryIndex, freq: 1); } } + + private void RemoveSpecialTerm(IndexedField field, EntryTermsReader reader, Slice termSlice, long entryToDelete, int termsPerEntryIndex) + { + ref var termLocation = ref CollectionsMarshal.GetValueRefOrAddDefault(field.Textual, termSlice, out var exists); + if (exists == false) + { + termLocation = field.Storage.Count; + field.Storage.AddByRef(new EntriesModifications(1)); + // We dont want to reclaim the term name + } + ref var term = ref field.Storage.GetAsRef(termLocation); + term.Removal(_entriesAllocator, entryToDelete, termsPerEntryIndex, reader.Frequency); + } public Dictionary GetIndexedFieldNamesByRootPage() { diff --git a/src/Corax/Querying/IndexSearcher.cs b/src/Corax/Querying/IndexSearcher.cs index a7f4a8efa431..2b170ab0eb11 100644 --- a/src/Corax/Querying/IndexSearcher.cs +++ b/src/Corax/Querying/IndexSearcher.cs @@ -38,9 +38,11 @@ public sealed unsafe partial class IndexSearcher : IDisposable private readonly IndexFieldsMapping _fieldMapping; private HashSet _nullTermsMarkers; + private HashSet _nonExistingTermsMarkers; private Tree _persistedDynamicTreeAnalyzer; private long? _numberOfEntries; public bool _nullTermsMarkersLoaded; + private bool _nonExistingTermsMarkersLoaded; /// /// When true no SIMD instruction will be used. Useful for checking that optimized algorithms behave in the same @@ -124,14 +126,10 @@ public EntryTermsReader GetEntryTermsReader(long id, ref Page p) if (_entryIdToLocation.TryGetValue(id, out var loc) == false) throw new InvalidOperationException("Unable to find entry id: " + id); - if (_nullTermsMarkersLoaded == false) - { - _nullTermsMarkersLoaded = true; - InitializeNullTermsMarkers(); - } + InitializeSpecialTermsMarkers(); var item = Container.MaybeGetFromSamePage(_transaction.LowLevelTransaction, ref p, loc); - return new EntryTermsReader(_transaction.LowLevelTransaction, _nullTermsMarkers, item.Address, item.Length, _dictionaryId); + return new EntryTermsReader(_transaction.LowLevelTransaction, _nullTermsMarkers, _nonExistingTermsMarkers, item.Address, item.Length, _dictionaryId); } internal void EncodeAndApplyAnalyzerForMultipleTerms(in FieldMetadata binding, ReadOnlySpan term, ref ContextBoundNativeList terms) @@ -584,26 +582,41 @@ public IncludeNonExistingMatch IncludeNonExistingMatch(in FieldM return new IncludeNonExistingMatch(this, inner, field, forward); } - private void InitializeNullTermsMarkers() + private void InitializeSpecialTermsMarkers() { - _nullTermsMarkers = new HashSet(); - InitNullPostingList(); - if (_nullPostingListsTree == null) - return; + if (_nullTermsMarkersLoaded == false) + { + _nullTermsMarkersLoaded = true; + _nullTermsMarkers = new HashSet(); + + InitNullPostingList(); + + if (_nullPostingListsTree != null) + LoadSpecialTermMarkers(_nullPostingListsTree, _nullTermsMarkers); + } - LoadNullTermMarkers(_nullPostingListsTree, _nullTermsMarkers); + if (_nonExistingTermsMarkersLoaded == false) + { + _nonExistingTermsMarkersLoaded = true; + _nonExistingTermsMarkers = new HashSet(); + + InitNonExistingPostingList(); + + if (_nonExistingPostingListsTree != null) + LoadSpecialTermMarkers(_nonExistingPostingListsTree, _nonExistingTermsMarkers); + } } - public static void LoadNullTermMarkers(Tree nullPostingList, HashSet nullTermsMarkers) + public static void LoadSpecialTermMarkers(Tree postingList, HashSet termsMarkers) { - using (var it = nullPostingList.Iterate(prefetch: false)) + using (var it = postingList.Iterate(prefetch: false)) { if (it.Seek(Slices.BeforeAllKeys)) { do { - (_, long nullTermId) = it.CreateReaderForCurrent().ReadStructure<(long, long)>(); - nullTermsMarkers.Add(nullTermId); + (_, long termId) = it.CreateReaderForCurrent().ReadStructure<(long, long)>(); + termsMarkers.Add(termId); } while (it.MoveNext()); } } diff --git a/src/Corax/Utils/EntryTermsReader.cs b/src/Corax/Utils/EntryTermsReader.cs index 22f18a94096a..2a8b2b044b82 100644 --- a/src/Corax/Utils/EntryTermsReader.cs +++ b/src/Corax/Utils/EntryTermsReader.cs @@ -77,6 +77,7 @@ public unsafe struct EntryTermsReader { private readonly LowLevelTransaction _llt; private readonly HashSet _nullTermsMarkers; + private readonly HashSet _nonExistingTermsMarkers; private readonly long _dicId; private byte* _cur; private readonly byte* _end, _start; @@ -96,12 +97,14 @@ public unsafe struct EntryTermsReader public bool IsRaw; public bool IsList; public bool IsNull; + public bool IsNonExisting; //rootPages has to be sorted. - public EntryTermsReader(LowLevelTransaction llt, HashSet nullTermsMarkers, byte* cur, int size, long dicId) + public EntryTermsReader(LowLevelTransaction llt, HashSet nullTermsMarkers, HashSet nonExistingTermsMarkers, byte* cur, int size, long dicId) { _llt = llt; _nullTermsMarkers = nullTermsMarkers; + _nonExistingTermsMarkers = nonExistingTermsMarkers; _start = _cur; _cur = cur; _start = cur; @@ -116,6 +119,7 @@ public EntryTermsReader(LowLevelTransaction llt, HashSet nullTermsMarkers, public bool FindNextStored(long fieldRootPage) { IsNull = false; + IsNonExisting = false; while (MoveNextStoredField()) { @@ -137,6 +141,7 @@ public bool FindNext(long fieldRootPage) public bool FindNextSpatial(long fieldRootPage) { IsNull = false; + IsNonExisting = false; while (MoveNextSpatial()) { @@ -153,6 +158,7 @@ public bool MoveNext() return false; IsNull = false; + IsNonExisting = false; var termContainerId = VariableSizeEncoding.Read(_cur, out var offset) + _prevTerm; _prevTerm = termContainerId; _cur += offset; @@ -197,6 +203,7 @@ public bool MoveNextStoredField() return false; IsNull = false; + IsNonExisting = false; var termContainerId = VariableSizeEncoding.Read(_cur, out var offset) + _prevTerm; _prevTerm = termContainerId; _cur += offset; @@ -228,10 +235,12 @@ private void HandleRegularTerm(long termContainerId) } IsNull = _nullTermsMarkers.Contains(TermId); + IsNonExisting = _nonExistingTermsMarkers.Contains(TermId); Container.Get(_llt, TermId, out var termItem); FieldRootPage = termItem.PageLevelMetadata; - if (IsNull == false) + + if (IsNull == false && IsNonExisting == false) { TermsReader.Set(Current, termItem, _dicId); } @@ -320,6 +329,7 @@ public void Reset() _prevLong = 0; _prevTerm = 0; IsNull = false; + IsNonExisting = false; } public string Debug(Indexing.IndexWriter w) diff --git a/test/SlowTests/Issues/RavenDB-22703.cs b/test/SlowTests/Issues/RavenDB-22703.cs index cfa6a1b90812..384f8976513c 100644 --- a/test/SlowTests/Issues/RavenDB-22703.cs +++ b/test/SlowTests/Issues/RavenDB-22703.cs @@ -1,6 +1,7 @@ using System.Linq; using FastTests; using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Operations; using Sparrow.Json.Parsing; using Tests.Infrastructure; using Xunit; @@ -51,6 +52,19 @@ public void TestQueryWithOrderByClauseAndNoWhereClause(Options options) .ToList(); Assert.Equal(4, res.Count); + + var deleteByQueryOp = new DeleteByQueryOperation("from 'Bars'"); + + store.Operations.Send(deleteByQueryOp); + + Indexes.WaitForIndexing(store); + + res = session.Query() + .OrderByDescending(b => b.Foo.BarBool) + .ThenByDescending(b => b.Foo.BarShort) + .ToList(); + + Assert.Equal(0, res.Count); } } }