From 35f6bc6edc8b5816dda17f3d6b15e42e258005e1 Mon Sep 17 00:00:00 2001 From: Bourne Shi Date: Wed, 16 Oct 2024 10:30:13 +0800 Subject: [PATCH] feat: support terms query --- .../ElasticsearchResponseHelper.cs | 17 +++- .../IElasticsearchClientProvider.cs | 30 +++---- .../Linq/GeneratorExpressionTreeVisitor.cs | 84 +++++++++++++++++-- .../ElasticsearchRepositoryTests.cs | 79 +++++++++++++++++ 4 files changed, 189 insertions(+), 21 deletions(-) diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs index ac177882..6c7b0b4c 100644 --- a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs +++ b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs @@ -6,6 +6,21 @@ public class ElasticsearchResponseHelper { public static string GetErrorMessage(IResponse response) { - return response.ServerError == null ? "Unknown error." : response.ServerError.ToString(); + if (response.ServerError == null) + { + if (response.OriginalException == null) + { + return "Unknown error."; + } + + if (response.OriginalException.InnerException == null) + { + return response.OriginalException.Message; + } + + return response.OriginalException.InnerException.Message; + } + + return response.ServerError.ToString(); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index 89b33684..f8e38ce7 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -20,21 +20,21 @@ public ElasticsearchClientProvider(IOptions options) { var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); - var settings = new ConnectionSettings(connectionPool); - // .DisableDirectStreaming(); - // .OnRequestCompleted(callDetails => - // { - // // Print Request DSL - // if (callDetails.RequestBodyInBytes != null) - // { - // Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); - // } - // // // Print Response Data - // // if (callDetails.ResponseBodyInBytes != null) - // // { - // // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); - // // } - // }); + var settings = new ConnectionSettings(connectionPool) + .DisableDirectStreaming() + .OnRequestCompleted(callDetails => + { + // Print Request DSL + if (callDetails.RequestBodyInBytes != null) + { + Console.WriteLine($"Request JSON: {Encoding.UTF8.GetString(callDetails.RequestBodyInBytes)}"); + } + // // Print Response Data + // if (callDetails.ResponseBodyInBytes != null) + // { + // Console.WriteLine($"Response JSON: {Encoding.UTF8.GetString(callDetails.ResponseBodyInBytes)}"); + // } + }); _elasticClient = new ElasticClient(settings); } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 7aa1750a..43dea89c 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using System.Reflection; using Elasticsearch.Net; using Remotion.Linq.Clauses; using Remotion.Linq.Clauses.Expressions; @@ -202,15 +203,41 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) case ContainsResultOperator containsResultOperator: Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - - if (containsResultOperator.Item.Type == typeof(Guid)) + + // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(expression.QueryModel.MainFromClause.FromExpression)); + Type itemType = containsResultOperator.Item.Type; + Type nonNullableType = Nullable.GetUnderlyingType(itemType) ?? itemType; + + // Handling different types + if (itemType == typeof(Guid) || nonNullableType == typeof(Guid)) { query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); } - - if (containsResultOperator.Item.Type == typeof(Guid?)) + else if (itemType == typeof(Guid?) || nonNullableType == typeof(Guid?)) + { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x?.ToString())); + } + else if (nonNullableType == typeof(int) || nonNullableType == typeof(int?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(long) || nonNullableType == typeof(long?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(double) || nonNullableType == typeof(double?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(DateTime) || nonNullableType == typeof(DateTime?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString("o"))); // ISO 8601 format + } + else if (nonNullableType == typeof(bool) || nonNullableType == typeof(bool?)) { + query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + } + else if (nonNullableType == typeof(string)) { + query = new TermsNode(PropertyName, (IEnumerable)Value); + } + else { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + throw new NotSupportedException($"Type {nonNullableType.Name} is not supported for Terms queries."); } QueryMap[expression] = ParseQuery(query); @@ -234,6 +261,11 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) QueryMap[expression].SubQueryPath = from; //from.ToLower(); QueryMap[expression].SubQueryFullPath = fullPath; // VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); + if (whereClause.Predicate is SubQueryExpression subQueryExpression) + { + // HandleNestedContains(subQueryExpression, tmp); + return base.VisitSubQuery(expression); + } BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; if (predicate.Left is BinaryExpression) { @@ -281,6 +313,48 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) return expression; } + // private void HandleNestedContains(SubQueryExpression subQueryExpression, Node parent) + // { + // if (subQueryExpression == null || parent == null) + // throw new ArgumentNullException("SubQueryExpression or parent Node cannot be null."); + // + // PropertyName = _propertyNameInferrerParser.Parser(GetFullPropertyPath(subQueryExpression)); + // PropertyType = Nullable.GetUnderlyingType(subQueryExpression.Type) ?? subQueryExpression.Type; + // Node query; + // if (PropertyType == typeof(Guid) || PropertyType == typeof(Guid)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else if (PropertyType == typeof(int) || PropertyType == typeof(int)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else if (PropertyType == typeof(string)) + // { + // query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + // } + // else + // { + // throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Contains queries."); + // } + // // parent.(query); + // QueryMap[subQueryExpression] = ParseQuery(query); + // } + + private string GetMemberName(Expression expression) + { + if (expression is MemberExpression memberExpression) + return memberExpression.Member.Name; + throw new InvalidOperationException("Expression does not represent a member access."); + } + + private object GetValueFromExpression(Expression expression) + { + if (expression is ConstantExpression constantExpression) + return constantExpression.Value; + throw new InvalidOperationException("Expression is not a constant."); + } + protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression) { return expression; diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index bee149dc..b633b799 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -593,6 +593,85 @@ public async Task GetList_Nested_Test() filterList.Count.ShouldBe(1); } + [Fact] + public async Task GetList_Terms_Test() + { + for (int i = 1; i <= 7; i++) + { + var blockIndex = new BlockIndex + { + Id = "block" + i, + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = DateTime.Now.AddDays(-10 + i), + LogEventCount = i, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + } + + List inputs = new List() + { + "BlockHash2", + "BlockHash3", + "BlockHash4" + }; + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + + var predicates = inputs + .Select(s => (Expression>)(info => info.BlockHash == s)) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable.Where(predicates).ToList(); + + var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); + filterList.Count.ShouldBe(3); + + List heights = new List() + { + 4, 5 + }; + Expression> mustQuery = item => heights.Contains(item.BlockHeight); + var filterList_heights = queryable.Where(mustQuery).ToList(); + filterList_heights.Count.ShouldBe(2); + } + + [Fact] + public async Task GetNestedList_Terms_Test() + { + //clear data for unit test + ClearTransactionIndex("AELF", 100, 110); + + Thread.Sleep(2000); + //Unit Test 14 + var transaction_100 = MockNewTransactionEtoData(100, false, "token_contract_address", "DonateResourceToken"); + var transaction_101 = MockNewTransactionEtoData(101, false, "", ""); + var transaction_103 = MockNewTransactionEtoData(103, false, "consensus_contract_address", "UpdateValue"); + var transaction_110 = MockNewTransactionEtoData(110, true, "consensus_contract_address", "UpdateTinyBlockInformation"); + await _transactionIndexRepository.AddAsync(transaction_100); + await _transactionIndexRepository.AddAsync(transaction_101); + await _transactionIndexRepository.AddAsync(transaction_103); + await _transactionIndexRepository.AddAsync(transaction_110); + + List inputs = new List() + { + 101, + 103 + }; + + // var predicates = inputs + // .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + // .Aggregate((prev, next) => prev.Or(next)); + + Expression> mustQuery = item => + item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); + + var queryable = await _transactionIndexRepository.GetQueryableAsync(); + // var filterList = queryable.Where(predicates).ToList(); + var filterList = queryable.Where(mustQuery).ToList(); + filterList.Count.ShouldBe(2); + } + [Fact] public async Task SubObjectQueryTest() {