Improved SelectMany() against value collections. Closes GH-2704. Closes

GH-2706

src/LinqTests/Acceptance/select_many.cs

@@ -628,7 +628,7 @@ select d.data as data from mt_temp_id_list3CTE as d
     public async Task can_query_with_where_clause_and_count_after_the_select_many()
     {
         var targets = Target.GenerateRandomData(1000).ToArray();
-        theStore.BulkInsert(targets);
+        await theStore.BulkInsertAsync(targets);
 
         await using var query = theStore.QuerySession();
 
@@ -644,7 +644,72 @@ public async Task can_query_with_where_clause_and_count_after_the_select_many()
         actual.ShouldBe(expected);
     }
 
+    [Fact]
+    public async Task select_many_on_value_collection_with_distinct_and_count()
+    {
+        // Addresses GH-2704
+        var targets = Target.GenerateRandomData(1000).ToArray();
+        await theStore.BulkInsertAsync(targets);
+
+        await using var query = theStore.QuerySession();
+        query.Logger = new TestOutputMartenLogger(_output);
+
+        var count = await query.Query<Target>().SelectMany(x => x.StringArray).Distinct().CountAsync();
+
+        var expected = targets.Where(x => x.StringArray != null).SelectMany(x => x.StringArray).Distinct().Count();
+
+        count.ShouldBe(expected);
+    }
+
+    [Fact]
+    public async Task select_many_on_value_collection_with_where_and_order_by()
+    {
+        // Addresses GH-2706
+        var targets = Target.GenerateRandomData(1000).ToArray();
+        await theStore.BulkInsertAsync(targets);
+
+        await using var query = theStore.QuerySession();
+        query.Logger = new TestOutputMartenLogger(_output);
+
+        var actual = await query.Query<Target>().Where(x => x.NumberArray != null)
+            .SelectMany(x => x.NumberArray)
+            .Where(x => x > 3)
+            .OrderBy(x => x)
+            .ToListAsync();
+
+        var expected = targets.Where(x => x.NumberArray != null)
+            .SelectMany(x => x.NumberArray)
+            .Where(x => x > 3)
+            .OrderBy(x => x).ToArray();
+
+        actual.ShouldHaveTheSameElementsAs(expected);
+
+    }
+
+    [Fact]
+    public async Task select_many_on_value_collection_with_where_and_order_by_on_strings()
+    {
+        // Addresses GH-2706
+        var targets = Target.GenerateRandomData(1000).ToArray();
+        await theStore.BulkInsertAsync(targets);
+
+        await using var query = theStore.QuerySession();
+        query.Logger = new TestOutputMartenLogger(_output);
+
+        var actual = await query.Query<Target>().Where(x => x.StringArray != null)
+            .SelectMany(x => x.StringArray)
+            .Where(x => x == "Green")
+            .OrderBy(x => x)
+            .ToListAsync();
+
+        var expected = targets.Where(x => x.StringArray != null)
+            .SelectMany(x => x.StringArray)
+            .Where(x => x == "Green")
+            .OrderBy(x => x).ToArray();
 
+        actual.ShouldHaveTheSameElementsAs(expected);
+
+    }
 
     public select_many(DefaultStoreFixture fixture, ITestOutputHelper output) : base(fixture)
     {

src/Marten/Linq/CollectionUsage.Compilation.cs

@@ -1,6 +1,7 @@
 #nullable enable
 
 using System;
+using System.Diagnostics;
 using JasperFx.Core.Reflection;
 using Marten.Exceptions;
 using Marten.Internal;
@@ -44,7 +45,7 @@ public Statement BuildTopStatement(IMartenSession session, IQueryableMemberColle
             }
         }
 
-        ProcessSingleValueModeIfAny(statement);
+        ProcessSingleValueModeIfAny(statement, session);
 
         compileNext(session, collection, statement);
 
@@ -62,12 +63,26 @@ public Statement BuildStatement(IMartenSession session, IQueryableMemberCollecti
     {
         var statement = new SelectorStatement
         {
-            SelectClause = selectClause ?? throw new ArgumentNullException(nameof(selectClause)),
-            Limit = _limit,
-            Offset = _offset,
-            IsDistinct = IsDistinct
+            SelectClause = selectClause ?? throw new ArgumentNullException(nameof(selectClause))
         };
 
+        ConfigureStatement(session, collection, statement);
+
+        if (IsDistinct)
+        {
+            statement.ApplySqlOperator("DISTINCT");
+        }
+
+        return statement;
+    }
+
+    internal Statement ConfigureStatement(IMartenSession session, IQueryableMemberCollection collection,
+        SelectorStatement statement)
+    {
+        statement.Limit = _limit;
+        statement.Offset = _offset;
+        statement.IsDistinct = IsDistinct;
+
         foreach (var ordering in OrderingExpressions)
             statement.Ordering.Expressions.Add(ordering.BuildExpression(collection));
 
@@ -88,15 +103,10 @@ public Statement BuildStatement(IMartenSession session, IQueryableMemberCollecti
             }
         }
 
-        ProcessSingleValueModeIfAny(statement);
+        ProcessSingleValueModeIfAny(statement, session);
 
         compileNext(session, collection, statement);
 
-        if (IsDistinct)
-        {
-            statement.ApplySqlOperator("DISTINCT");
-        }
-
         return statement.Top();
     }
 
@@ -162,18 +172,14 @@ public void CompileSelectMany(IMartenSession session, CollectionUsage parent,
 
 
         // THINK THIS IS TOO SOON. MUCH OF THE LOGIC NEEDS TO GO IN THIS INSTEAD!!!
-        var childStatement = collectionMember.BuildChildStatement(this, session, parentStatement);
-
-        childStatement.IsDistinct = IsDistinct;
-        childStatement.Limit = _limit;
-        childStatement.Offset = _offset;
+        var childStatement = collectionMember.BuildSelectManyStatement(this, session, parentStatement);
 
         if (IsDistinct)
         {
             if (childStatement.SelectClause is IScalarSelectClause c)
             {
                 c.ApplyOperator("DISTINCT");
-                parentStatement.InsertAfter(childStatement.SelectorStatement());
+                parentStatement.AddToEnd(childStatement.Top());
             }
             else if (childStatement.SelectClause is ICountClause count)
             {
@@ -195,7 +201,7 @@ public void CompileSelectMany(IMartenSession session, CollectionUsage parent,
         }
         else
         {
-            parentStatement.InsertAfter(childStatement);
+            parentStatement.AddToEnd(childStatement.Top());
         }
 
         compileNext(session, collectionMember as IQueryableMemberCollection, childStatement);
@@ -212,7 +218,7 @@ public void CompileAsChild(CollectionUsage parent, SelectorStatement parentState
         }
     }
 
-    internal void ProcessSingleValueModeIfAny(SelectorStatement statement)
+    internal void ProcessSingleValueModeIfAny(SelectorStatement statement, IMartenSession session)
     {
         if (IsAny || SingleValueMode == Marten.Linq.Parsing.SingleValueMode.Any)
         {
@@ -260,21 +266,33 @@ internal void ProcessSingleValueModeIfAny(SelectorStatement statement)
 
                     if (statement.IsDistinct)
                     {
-                        throw new NotImplementedException("Not yet");
-                        // statement.ConvertToCommonTableExpression(_session);
-                        // statement = new CountStatement<int>(CurrentStatement);
+                        statement.ConvertToCommonTableExpression(session);
+                        var count = new SelectorStatement
+                        {
+                            SelectClause = new CountClause<int>(statement.ExportName)
+                        };
+
+                        statement.AddToEnd(count);
                     }
 
                     statement.SelectClause = new CountClause<int>(statement.SelectClause.FromObject);
 
                     break;
 
                 case Marten.Linq.Parsing.SingleValueMode.LongCount:
+                    // Invalid to be using OrderBy() while also using Count() in
+                    // PostgreSQL. Thank you Hot Chocolate.
+                    statement.Ordering.Expressions.Clear();
+
                     if (statement.IsDistinct)
                     {
-                        throw new NotImplementedException("Not yet");
-                        // statement.ConvertToCommonTableExpression(_session);
-                        // statement = new CountStatement<long>(CurrentStatement);
+                        statement.ConvertToCommonTableExpression(session);
+                        var count = new SelectorStatement
+                        {
+                            SelectClause = new CountClause<long>(statement.ExportName)
+                        };
+
+                        statement.AddToEnd(count);
                     }
 
                     statement.SelectClause = new CountClause<long>(statement.SelectClause.FromObject);

src/Marten/Linq/Members/ChildCollectionMember.cs

@@ -55,7 +55,7 @@ public ISqlFragment ParseWhereForAny(Expression body, IReadOnlyStoreOptions opti
 
     public string ExplodeLocator { get; }
 
-    public SelectorStatement BuildChildStatement(CollectionUsage collectionUsage, IMartenSession session,
+    public SelectorStatement BuildSelectManyStatement(CollectionUsage collectionUsage, IMartenSession session,
         SelectorStatement parentStatement)
     {
         var selectClause =

src/Marten/Linq/Members/DictionaryMember.cs

@@ -60,7 +60,7 @@ public override void PlaceValueInDictionaryForContainment(Dictionary<string, obj
 
     string ICollectionMember.ArrayLocator => throw new NotImplementedException();
 
-    SelectorStatement ICollectionMember.BuildChildStatement(CollectionUsage collectionUsage, IMartenSession session,
+    SelectorStatement ICollectionMember.BuildSelectManyStatement(CollectionUsage collectionUsage, IMartenSession session,
         SelectorStatement parentStatement)
     {
         throw new NotImplementedException();

src/Marten/Linq/Members/DuplicatedArrayField.cs

@@ -7,6 +7,7 @@
 using JasperFx.Core.Reflection;
 using Marten.Exceptions;
 using Marten.Internal;
+using Marten.Linq.Members.ValueCollections;
 using Marten.Linq.Parsing;
 using Marten.Linq.SqlGeneration;
 using Marten.Linq.SqlGeneration.Filters;
@@ -44,15 +45,15 @@ public DuplicatedArrayField(EnumStorage enumStorage, QueryableMember innerMember
     public string ArrayLocator => TypedLocator;
     public IQueryableMember ElementMember { get; }
 
-    public SelectorStatement BuildChildStatement(CollectionUsage collectionUsage, IMartenSession session,
+    public SelectorStatement BuildSelectManyStatement(CollectionUsage collectionUsage, IMartenSession session,
         SelectorStatement parentStatement)
     {
         var statement = ElementType == typeof(string)
             ? new ScalarSelectManyStringStatement(parentStatement)
             : typeof(ScalarSelectManyStatement<>).CloseAndBuildAs<SelectorStatement>(parentStatement,
                 session.Serializer, ElementType);
 
-        collectionUsage.ProcessSingleValueModeIfAny(statement);
+        collectionUsage.ProcessSingleValueModeIfAny(statement, session);
 
         return statement;
     }

src/Marten/Linq/Members/ICollectionMember.cs

@@ -17,7 +17,7 @@ public interface ICollectionMember: IQueryableMember
 
     string ArrayLocator { get;  }
 
-    SelectorStatement BuildChildStatement(CollectionUsage collectionUsage, IMartenSession session,
+    SelectorStatement BuildSelectManyStatement(CollectionUsage collectionUsage, IMartenSession session,
         SelectorStatement parentStatement);
 
     ISelectClause BuildSelectClauseForExplosion(string fromObject);

src/Marten/Linq/Members/IQueryableMember.cs

@@ -20,12 +20,12 @@ public interface IQueryableMember: ISqlFragment
     /// <summary>
     ///     Postgresql locator that also casts the raw string data to the proper Postgresql type
     /// </summary>
-    string TypedLocator { get; }
+    string TypedLocator { get;}
 
     /// <summary>
     ///     Postgresql locator that returns the raw string value within the JSONB document
     /// </summary>
-    string RawLocator { get; }
+    string RawLocator { get;}
 
     /// <summary>
     ///     Locate the data for this field as JSONB

src/Marten/Linq/Members/QueryableMember.cs

@@ -26,6 +26,8 @@ public abstract class QueryableMember: IQueryableMember, IHasChildrenMembers
     /// <param name="member"></param>
     protected QueryableMember(IQueryableMember parent, Casing casing, MemberInfo member)
     {
+        if (parent == null) throw new ArgumentNullException(nameof(parent));
+
         Member = member;
         MemberType = member.GetMemberType();
 
@@ -80,7 +82,7 @@ public virtual void ReplaceMember(MemberInfo member, IQueryableMember queryableM
 
     public IQueryableMember[] Ancestors { get; }
 
-    public string RawLocator { get; protected set; }
+    public string RawLocator { get; protected internal set; }
 
 
     public string NullTestLocator
@@ -89,7 +91,7 @@ public string NullTestLocator
         protected set => _nullTestLocator = value;
     }
 
-    public string TypedLocator { get; protected set; }
+    public string TypedLocator { get; protected internal set; }
 
     /// <summary>
     ///     Locate the data for this field as JSONB