From cc6bf13e316ea722576bb1e3ca4057bd93815f5d Mon Sep 17 00:00:00 2001 From: "Jeremy D. Miller" Date: Tue, 5 Dec 2023 15:05:11 -0600 Subject: [PATCH] Allowing users to add LINQ order by operators with literal SQL. Closes GH-2843 --- docs/configuration/cli.md | 106 ------------------ docs/configuration/ioc.md | 59 ---------- docs/documents/plv8.md | 3 +- src/LinqTests/Acceptance/order_by_clauses.cs | 1 + src/LinqTests/Acceptance/order_by_sql.cs | 58 ++++++++++ .../Bug_2563_sub_collection_queries_and_OR.cs | 20 ++-- src/Marten.Testing/Examples/LinqExamples.cs | 6 +- src/Marten/Linq/MartenLinqQueryable.cs | 1 + src/Marten/Linq/Members/DuplicatedField.cs | 3 +- .../Linq/Parsing/Operators/OperatorLibrary.cs | 11 +- .../Parsing/Operators/OrderBySqlOperator.cs | 36 ++++++ src/Marten/Linq/Parsing/Operators/Ordering.cs | 12 ++ src/Marten/Marten.csproj | 1 - src/Marten/QueryableExtensions.cs | 34 ++++++ 14 files changed, 166 insertions(+), 185 deletions(-) create mode 100644 src/LinqTests/Acceptance/order_by_sql.cs create mode 100644 src/Marten/Linq/Parsing/Operators/OrderBySqlOperator.cs diff --git a/docs/configuration/cli.md b/docs/configuration/cli.md index 609f3348bc..e69de29bb2 100644 --- a/docs/configuration/cli.md +++ b/docs/configuration/cli.md @@ -1,106 +0,0 @@ -# Command Line Tooling - -::: warning -The usage of Marten.CommandLine shown in this document is only valid for applications bootstrapped with the -[generic host builder](https://docs.microsoft.com/en-us/aspnet/core/fundamentals/host/generic-host) with Marten registered in the application's IoC container. -::: - -There is a separate NuGet package called _Marten.CommandLine_ that can be used to quickly add command-line tooling directly to -your .Net Core application that uses Marten. _Marten.CommandLine_ is an extension library to [Oakton](https://jasperfx.github.io/oakton) that -is the actual command line parser in this case. - -To use the expanded command line options to a .NET application, add a reference to the _Marten.CommandLine_ Nuget and add this line of code to your `Program.cs`: - - - -```cs -var builder = WebApplication.CreateBuilder(args); - -// Easiest to just do this right after creating builder -// Must be done before calling builder.Build() at least -builder.Host.ApplyOaktonExtensions(); -``` -snippet source | anchor - - -And finally, use Oakton as the command line parser and executor by replacing `App.Run()` as the last line of code in your -`Program.cs` file: - - - -```cs -// Instead of App.Run(), use the app.RunOaktonCommands(args) -// as the last line of your Program.cs file -return await app.RunOaktonCommands(args); -``` -snippet source | anchor - - -Once the _Marten.CommandLine_ Nuget is installed and Oakton is handling your command line parsing, you should be able to see the Marten commands by typing `dotnet run -- help` in the command line terminal of your choice at the root of your project: - -```bash - ---------------------------------------------------------------------------------------------------------- - Available commands: - ---------------------------------------------------------------------------------------------------------- - check-env -> Execute all environment checks against the application - describe -> Writes out a description of your running application to either the console or a file - help -> list all the available commands - marten-apply -> Applies all outstanding changes to the database based on the current configuration - marten-assert -> Assert that the existing database matches the current Marten configuration - marten-dump -> Dumps the entire DDL for the configured Marten database - marten-patch -> Evaluates the current configuration against the database and writes a patch and drop file if there are any differences - projections -> Rebuilds all projections of specified kind - run -> Start and run this .Net application - ---------------------------------------------------------------------------------------------------------- -``` - -If you're not using the dotnet CLI yet, you'd just need to compile your new console application like you've always done and call the exe directly. If you're familiar with the *nix style of command-line interfaces ala Git, you should feel right at home with the command line usage in Marten. - -For the sake of usability, let's say that you stick a file named "marten.cmd" (or the *nix shell file equivalent) at the root of your codebase like so: - -```bash -dotnet run --project src/MyConsoleApp %* -``` - -All the example above does is delegate any arguments to your console application. Once you have that file, some sample usages are shown below: - -Assert that the database matches the current database. This command will fail if there are differences - -```bash -marten marten-assert --log log.txt -``` - -This command tries to update the database to reflect the application configuration - -```bash -marten marten-apply --log log.txt -``` - -This dumps a single file named "database.sql" with all the DDL necessary to build the database to -match the application configuration - -```bash -marten marten-dump database.sql -``` - -This dumps the DDL to separate files per document -type to a folder named "scripts" - -```bash -marten marten-dump scripts --by-type -``` - -Create a patch file called "patch1.sql" and -the corresponding rollback file "patch.drop.sql" if any -differences are found between the application configuration -and the database - -```bash -marten marten-patch patch1.sql --drop patch1.drop.sql -``` - -In all cases, the commands expose usage help through "marten help [command]." Each of the commands also exposes a "--conn" (or "-c" if you prefer) flag to override the database connection string and a "--log" flag to record all the command output to a file. - -## Projections Support - -See [the Async Daemon documentation](/events/projections/async-daemon.md) for more information about the newly improved `projections` command. diff --git a/docs/configuration/ioc.md b/docs/configuration/ioc.md index 11d2f3c400..e69de29bb2 100644 --- a/docs/configuration/ioc.md +++ b/docs/configuration/ioc.md @@ -1,59 +0,0 @@ -# Custom IoC Integration - -::: tip -The Marten team recommends using the `IServiceCollection.AddMarten()` extension method -for IoC integration out of the box. -::: - -The Marten team has striven to make the library perfectly usable without the usage of an IoC container, but you may still want to -use an IoC container specifically to manage dependencies and the life cycle of Marten objects. -While the `IServiceCollection.AddMarten()` method is the recommended way to integrate Marten -into an IoC container, you can certainly recreate that functionality in the IoC container -of your choice. - -::: tip INFO -Lamar supports the .Net Core abstractions for IoC service registrations, so you *could* happily -use the `AddMarten()` method directly with Lamar as well. -::: - -Using [Lamar](https://jasperfx.github.io/lamar) as the example container, we recommend registering Marten something like this: - - - -```cs -public class MartenServices : ServiceRegistry -{ - public MartenServices() - { - ForSingletonOf().Use(c => - { - return DocumentStore.For(options => - { - options.Connection("your connection string"); - options.AutoCreateSchemaObjects = AutoCreate.None; - - // other Marten configuration options - }); - }); - - // Register IDocumentSession as Scoped - For() - .Use(c => c.GetInstance().LightweightSession()) - .Scoped(); - - // Register IQuerySession as Scoped - For() - .Use(c => c.GetInstance().QuerySession()) - .Scoped(); - } -} -``` -snippet source | anchor - - -There are really only two key points here: - -1. There should only be one `IDocumentStore` object instance created in your application, so I scoped it as a "Singleton" in the StructureMap container -1. The `IDocumentSession` service that you use to read and write documents should be scoped as "one per transaction." In typical usage, this - ends up meaning that an `IDocumentSession` should be scoped to a single HTTP request in web applications or a single message being handled in service - bus applications. diff --git a/docs/documents/plv8.md b/docs/documents/plv8.md index df5b15ae49..c5bea00b0a 100644 --- a/docs/documents/plv8.md +++ b/docs/documents/plv8.md @@ -303,8 +303,7 @@ The `Patch.Remove()` operation removes the given item from a child collection: ```cs -[Fact] -public void remove_primitive_element() +[Fact]public void remove_primitive_element() { var random = new Random(); var target = Target.Random(); diff --git a/src/LinqTests/Acceptance/order_by_clauses.cs b/src/LinqTests/Acceptance/order_by_clauses.cs index 05cc313e4c..1531bdb9a0 100644 --- a/src/LinqTests/Acceptance/order_by_clauses.cs +++ b/src/LinqTests/Acceptance/order_by_clauses.cs @@ -22,4 +22,5 @@ static order_by_clauses() ordered(t => t.OrderBy(x => x.String).Skip(2)); ordered(t => t.OrderBy(x => x.String).Take(2).Skip(2)); } + } diff --git a/src/LinqTests/Acceptance/order_by_sql.cs b/src/LinqTests/Acceptance/order_by_sql.cs new file mode 100644 index 0000000000..96f17df93e --- /dev/null +++ b/src/LinqTests/Acceptance/order_by_sql.cs @@ -0,0 +1,58 @@ +using System.Linq; +using System.Threading.Tasks; +using Marten; +using Marten.Testing.Documents; +using Marten.Testing.Harness; +using Xunit.Abstractions; + +namespace LinqTests.Acceptance; + +public class order_by_sql : OneOffConfigurationsContext +{ + private readonly ITestOutputHelper _output; + + public order_by_sql(ITestOutputHelper output) + { + _output = output; + } + + [Fact] + public async Task sort_by_literal_sql() + { + StoreOptions(x => + { + x.Schema.For() + .Duplicate(x => x.String) + .Duplicate(x => x.AnotherString); + }); + + var targets = Target.GenerateRandomData(100).ToArray(); + await theStore.BulkInsertAsync(targets); + + theSession.Logger = new TestOutputMartenLogger(_output); + + var expected = await theSession + .Query() + .OrderBy(x => x.String) + .ThenByDescending(x => x.AnotherString) + .Select(x => x.Id) + .ToListAsync(); + + var command = theSession + .Query() + .OrderBySql("string") + .ThenBySql("another_string desc") + .Select(x => x.Id).ToCommand(); + + _output.WriteLine(command.CommandText); + + var actual = await theSession + .Query() + .OrderBySql("string") + .ThenBySql("another_string desc") + .Select(x => x.Id) + .ToListAsync(); + + actual.ShouldHaveTheSameElementsAs(expected); + } +} diff --git a/src/LinqTests/Bugs/Bug_2563_sub_collection_queries_and_OR.cs b/src/LinqTests/Bugs/Bug_2563_sub_collection_queries_and_OR.cs index 7c0cecb725..bc3aa93071 100644 --- a/src/LinqTests/Bugs/Bug_2563_sub_collection_queries_and_OR.cs +++ b/src/LinqTests/Bugs/Bug_2563_sub_collection_queries_and_OR.cs @@ -19,34 +19,34 @@ public Bug_2563_sub_collection_queries_and_OR(ITestOutputHelper output) [Fact] public async Task get_distinct_number() { - theStore.Options.Schema.For() + theStore.Options.Schema.For() .Duplicate(x => x.UserIds); - theSession.Store(new Target {Id = 1, IsPublic = false, UserIds = new [] { 1, 2, 3, 4, 5, 6 }}); - theSession.Store(new Target {Id = 2, IsPublic = false, UserIds = new int[] { }}); - theSession.Store(new Target {Id = 3, IsPublic = true, UserIds = new [] { 1, 2, 3 }}); - theSession.Store(new Target {Id = 4, IsPublic = true, UserIds = new [] { 1, 2, 6 }}); - theSession.Store(new Target {Id = 5, IsPublic = false, UserIds = new [] { 4, 5, 6 }}); - theSession.Store(new Target {Id = 6, IsPublic = true, UserIds = new [] { 10 }}); + theSession.Store(new Bug2563Target {Id = 1, IsPublic = false, UserIds = new [] { 1, 2, 3, 4, 5, 6 }}); + theSession.Store(new Bug2563Target {Id = 2, IsPublic = false, UserIds = new int[] { }}); + theSession.Store(new Bug2563Target {Id = 3, IsPublic = true, UserIds = new [] { 1, 2, 3 }}); + theSession.Store(new Bug2563Target {Id = 4, IsPublic = true, UserIds = new [] { 1, 2, 6 }}); + theSession.Store(new Bug2563Target {Id = 5, IsPublic = false, UserIds = new [] { 4, 5, 6 }}); + theSession.Store(new Bug2563Target {Id = 6, IsPublic = true, UserIds = new [] { 10 }}); await theSession.SaveChangesAsync(); theSession.Logger = new TestOutputMartenLogger(_output); - var result1 = await theSession.Query() + var result1 = await theSession.Query() .Where(x => x.IsPublic == false || x.UserIds.Contains(10)) .ToListAsync(); result1.Count.ShouldBeEquivalentTo(4); // This should pass without any error as the query will return results - var result2 = await theSession.Query().Where(x => x.IsPublic || x.UserIds.Contains(5)).ToListAsync(); + var result2 = await theSession.Query().Where(x => x.IsPublic || x.UserIds.Contains(5)).ToListAsync(); result2.ShouldContain(x => x.Id == 1); result2.ShouldContain(x => x.Id == 5); } - public class Target + public class Bug2563Target { public int Id { get; set; } diff --git a/src/Marten.Testing/Examples/LinqExamples.cs b/src/Marten.Testing/Examples/LinqExamples.cs index 602cf87da8..7b7e904f73 100644 --- a/src/Marten.Testing/Examples/LinqExamples.cs +++ b/src/Marten.Testing/Examples/LinqExamples.cs @@ -77,6 +77,7 @@ public void case_insensitive_string_fields(IDocumentSession session) session.Query().Where(x => x.String.Contains("soMeThiNg", StringComparison.OrdinalIgnoreCase)); session.Query().Where(x => x.String.Equals("ThE SaMe ThInG", StringComparison.OrdinalIgnoreCase)); + } #endregion @@ -92,6 +93,9 @@ public void order_by(IDocumentSession session) // You can use multiple order by's session.Query().OrderBy(x => x.Date).ThenBy(x => x.Number); + + // If you're brave, you can even use raw SQL literals as of Marten v7! + session.Query().OrderBySql("substring(d.data -> 'String', 1, 2)"); } #endregion @@ -199,4 +203,4 @@ public async Task sample_aggregation_operations(IQuerySession session) } #endregion -} \ No newline at end of file +} diff --git a/src/Marten/Linq/MartenLinqQueryable.cs b/src/Marten/Linq/MartenLinqQueryable.cs index a2bfb4e544..2b0c39da57 100644 --- a/src/Marten/Linq/MartenLinqQueryable.cs +++ b/src/Marten/Linq/MartenLinqQueryable.cs @@ -13,6 +13,7 @@ using Marten.Internal.Storage; using Marten.Linq.Includes; using Marten.Linq.Parsing; +using Marten.Linq.Parsing.Operators; using Marten.Linq.QueryHandlers; using Marten.Services; using Npgsql; diff --git a/src/Marten/Linq/Members/DuplicatedField.cs b/src/Marten/Linq/Members/DuplicatedField.cs index 10f6079508..44b607f0c0 100644 --- a/src/Marten/Linq/Members/DuplicatedField.cs +++ b/src/Marten/Linq/Members/DuplicatedField.cs @@ -99,8 +99,7 @@ public DuplicatedField(EnumStorage enumStorage, QueryableMember innerMember, public string JsonPathSegment => throw new NotSupportedException(); public string BuildOrderingExpression(Ordering ordering, CasingRule casingRule) { - // TODO -- memoize or intern this. Watch if this is a string!!! - if (ordering.Direction == OrderingDirection.Desc) return "d.id desc"; + if (ordering.Direction == OrderingDirection.Desc) return $"{TypedLocator} desc"; return TypedLocator; } diff --git a/src/Marten/Linq/Parsing/Operators/OperatorLibrary.cs b/src/Marten/Linq/Parsing/Operators/OperatorLibrary.cs index 24d12cc0e0..c07d649129 100644 --- a/src/Marten/Linq/Parsing/Operators/OperatorLibrary.cs +++ b/src/Marten/Linq/Parsing/Operators/OperatorLibrary.cs @@ -16,11 +16,11 @@ public OperatorLibrary() Add(); Add(); - AddOrdering("OrderBy", OrderingDirection.Asc); - AddOrdering("ThenBy", OrderingDirection.Asc); + AddOrdering(nameof(QueryableExtensions.OrderBy), OrderingDirection.Asc); + AddOrdering(nameof(QueryableExtensions.ThenBy), OrderingDirection.Asc); - AddOrdering("OrderByDescending", OrderingDirection.Desc); - AddOrdering("ThenByDescending", OrderingDirection.Desc); + AddOrdering(nameof(QueryableExtensions.OrderByDescending), OrderingDirection.Desc); + AddOrdering(nameof(QueryableExtensions.ThenByDescending), OrderingDirection.Desc); Add(); Add(); @@ -28,6 +28,9 @@ public OperatorLibrary() Add(); Add(); + Add(); + Add(); + foreach (var mode in Enum.GetValues()) addSingleValueMode(mode); } diff --git a/src/Marten/Linq/Parsing/Operators/OrderBySqlOperator.cs b/src/Marten/Linq/Parsing/Operators/OrderBySqlOperator.cs new file mode 100644 index 0000000000..ac5d5d414a --- /dev/null +++ b/src/Marten/Linq/Parsing/Operators/OrderBySqlOperator.cs @@ -0,0 +1,36 @@ +using System.Linq; +using System.Linq.Expressions; + +namespace Marten.Linq.Parsing.Operators; + +internal class OrderBySqlOperator : LinqOperator +{ + public OrderBySqlOperator() : base(nameof(QueryableExtensions.OrderBySql)) + { + } + + public override void Apply(ILinqQuery query, MethodCallExpression expression) + { + var sql = expression.Arguments.Last().ReduceToConstant(); + var usage = query.CollectionUsageFor(expression); + var ordering = new Ordering((string)sql.Value); + + usage.OrderingExpressions.Insert(0, ordering); + } +} + +internal class ThenBySqlOperator : LinqOperator +{ + public ThenBySqlOperator() : base(nameof(QueryableExtensions.ThenBySql)) + { + } + + public override void Apply(ILinqQuery query, MethodCallExpression expression) + { + var sql = expression.Arguments.Last().ReduceToConstant(); + var usage = query.CollectionUsageFor(expression); + var ordering = new Ordering((string)sql.Value); + + usage.OrderingExpressions.Insert(0, ordering); + } +} diff --git a/src/Marten/Linq/Parsing/Operators/Ordering.cs b/src/Marten/Linq/Parsing/Operators/Ordering.cs index f1db6c032b..f8132ea298 100644 --- a/src/Marten/Linq/Parsing/Operators/Ordering.cs +++ b/src/Marten/Linq/Parsing/Operators/Ordering.cs @@ -1,16 +1,26 @@ using System.Linq.Expressions; +using JasperFx.Core; using Marten.Linq.Members; namespace Marten.Linq.Parsing.Operators; public class Ordering { + private readonly string _literal; + public Ordering(Expression expression, OrderingDirection direction) { Expression = expression; Direction = direction; } + public Ordering(string literal) + { + _literal = literal; + } + + public string Literal => _literal; + public Expression Expression { get; } public OrderingDirection Direction { get; } @@ -25,6 +35,8 @@ public Ordering(Expression expression, OrderingDirection direction) public string BuildExpression(IQueryableMemberCollection collection) { + if (_literal.IsNotEmpty()) return _literal; + var member = collection.MemberFor(Expression, "Invalid OrderBy() expression"); return member.BuildOrderingExpression(this, CasingRule); diff --git a/src/Marten/Marten.csproj b/src/Marten/Marten.csproj index ee47e04090..fc1ed29b92 100644 --- a/src/Marten/Marten.csproj +++ b/src/Marten/Marten.csproj @@ -44,7 +44,6 @@ - diff --git a/src/Marten/QueryableExtensions.cs b/src/Marten/QueryableExtensions.cs index 994cc5496f..44a59afcc4 100644 --- a/src/Marten/QueryableExtensions.cs +++ b/src/Marten/QueryableExtensions.cs @@ -4,10 +4,12 @@ using System.IO; using System.Linq; using System.Linq.Expressions; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using JasperFx.Core.Reflection; using Marten.Linq; +using Marten.Linq.Parsing.Operators; using Marten.Services.BatchQuerying; using Npgsql; @@ -699,4 +701,36 @@ private static LambdaExpression CompileOrderBy(string property, out Type targ } #endregion + + private static MethodInfo _orderBySqlMethod = typeof(QueryableExtensions).GetMethod(nameof(OrderBySql), + BindingFlags.Public | BindingFlags.Static | BindingFlags.NonPublic); + + private static MethodInfo _thenBySqlMethod = typeof(QueryableExtensions).GetMethod(nameof(ThenBySql), + BindingFlags.Public | BindingFlags.Static | BindingFlags.NonPublic); + + /// + /// Supply literal SQL fragments to be placed in the generated SQL for this LINQ query. + /// You can supply the "desc" suffix here + /// + /// + /// + /// + public static IQueryable OrderBySql(this IQueryable queryable, string sql) + { + return queryable.Provider.CreateQuery(Expression.Call(null, _orderBySqlMethod.MakeGenericMethod(typeof(T)), queryable.Expression, + Expression.Constant(sql))); + } + + /// + /// Supply literal SQL fragments to be placed in the generated SQL for this LINQ query + /// You can supply the "desc" suffix here + /// + /// + /// + /// + public static IQueryable ThenBySql(this IQueryable queryable, string sql) + { + return queryable.Provider.CreateQuery(Expression.Call(null, _thenBySqlMethod.MakeGenericMethod(typeof(T)), queryable.Expression, + Expression.Constant(sql))); + } }