Feature: Adjust the maximum number of queries (#28)

* feat: fix changing non-concurrent collections * feat: adjust max size * feat: add issue template * feat: improve response error * feat: throw bulk error --------- Co-authored-by: jason wang <[email protected]>
AElfProject · Aug 30, 2024 · 194d023 · 194d023
1 parent e8a631b
commit 194d023
Show file tree

Hide file tree

Showing 10 changed files with 115 additions and 38 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,33 @@
+---
+name: 👾 Bug Report
+about: Report a bug or issue with the project.
+title: ''
+labels: 'bug'
+assignees: ''
+
+---
+
+### Description
+A clear and concise description of what the bug is.
+
+### Steps To Reproduce
+1. Log in...
+2. Ensure that...
+3. Allow a long period of inactivity to pass...
+4. Observe that...
+5. Attempt to log in...
+
+### Current Behavior
+- After the period of inactivity...
+- When the user tries to log in using another method...
+- This causes a bug due to...
+
+### Expected Behavior
+- After a long period of inactivity...
+- When a user logs in successfully...
+- This ensures that only...
+
+### Environment
+- Platform: PC
+- Node: v18.18.0
+- Browser: Chrome 126.0.6478.56
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1,10 @@
+blank_issues_enabled: false
+issue_template:
+  - name: 👾 Bug Report
+    description: Report a bug or issue with the project.
+    labels: ["bug"]
+    template: bug_report.md
+  - name: 💡 Feature Request
+    description: Create a new ticket for a new feature request.
+    labels: ["enhancement"]
+    template: feature_request.md
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,29 @@
+---
+name: 💡 Feature Request
+about: Create a new ticket for a new feature request
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+### Expected Behavior
+Describe the expected behavior here.
+
+### Specifications
+As a `user`, I would like to `action` so that `reason`.
+
+**Features:**
+- describe feature details here.
+
+**Development Tasks:**
+- [ ] Task 1
+- [ ] Task 2
+
+### Dependencies
+List any dependencies that are required for this feature by providing links to the issues or repositories.
+
+### References
+List any references that are related to this feature request.
+
+.github/workflows/nuget-publish.yml
diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs
@@ -0,0 +1,11 @@
+using Nest;
+
+namespace AElf.EntityMapping.Elasticsearch;
+
+public class ElasticsearchResponseHelper
+{
+    public static string GetErrorMessage(IResponse response)
+    {
+        return response.ServerError == null ? "Unknown error." : response.ServerError.ToString();
+    }
+}
diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs
@@ -74,17 +74,7 @@ public IEnumerable<T> ExecuteCollection<T>(QueryModel queryModel)
 
                 if (queryAggregator.Take != null)
                 {
-                    var take = queryAggregator.Take.Value;
-                    var skip = queryAggregator.Skip ?? 0;
-
-                    if (skip + take > ElasticQueryLimit)
-                    {
-                        var exceedCount = skip + take - ElasticQueryLimit;
-                        take -= exceedCount;
-                    }
-
-                    descriptor.Take(take);
-                    descriptor.Size(take);
+                    descriptor.Size(queryAggregator.Take.Value);
                 }
 
                 if (queryAggregator.After != null)
@@ -152,7 +142,7 @@ public IEnumerable<T> ExecuteCollection<T>(QueryModel queryModel)
             if (!documents.IsValid)
             {
                 throw new ElasticsearchException($"Search document failed at index {index} :" +
-                                                 documents.ServerError.Error.Reason);
+                                                 ElasticsearchResponseHelper.GetErrorMessage(documents));
             }
 
             if (queryModel.SelectClause?.Selector is MemberExpression)
@@ -247,7 +237,7 @@ public T ExecuteScalar<T>(QueryModel queryModel)
                     if (!response.IsValid)
                     {
                         throw new ElasticsearchException(
-                            $"Count Document failed at index {index} :{response.ServerError.Error.Reason}");
+                            $"Count Document failed at index {index} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
                     }
                     var result = response.Count;
                     /*if (result > ElasticQueryLimit)

diff --git a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs
@@ -8,5 +8,5 @@ public class ElasticsearchOptions
     public int NumberOfShards { get; set; } = 1;
     public int NumberOfReplicas { get; set; } = 1;
     public Refresh Refresh { get; set; } = Refresh.False;
-    public int MaxResultWindow { get; set; } = int.MaxValue;
+    public int MaxResultWindow { get; set; } = 10000;
 }
diff --git a/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs b/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs
@@ -110,7 +110,7 @@ public async Task AddAsync(TEntity model, string collectionName = null, Cancella
         if (result.IsValid)
             return;
         throw new ElasticsearchException(
-            $"Insert Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {result.ServerError.Error.Reason}");
+            $"Insert Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {ElasticsearchResponseHelper.GetErrorMessage(result)}");
     }
 
     public async Task AddOrUpdateAsync(TEntity model, string collectionName = null,
@@ -132,7 +132,7 @@ public async Task AddOrUpdateAsync(TEntity model, string collectionName = null,
             if (result.IsValid)
                 return;
             throw new ElasticsearchException(
-                $"Update Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {result.ServerError.Error.Reason}");
+                $"Update Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {ElasticsearchResponseHelper.GetErrorMessage(result)}");
         }
         else
         {
@@ -145,7 +145,7 @@ await client.IndexAsync(model, ss => ss.Index(indexName).Refresh(_elasticsearchO
             if (result.IsValid)
                 return;
             throw new ElasticsearchException(
-                $"Insert Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {result.ServerError.Error.Reason}");
+                $"Insert Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {ElasticsearchResponseHelper.GetErrorMessage(result)}");
         }
     }
 
@@ -204,8 +204,10 @@ private async Task BulkAddAsync(IElasticClient client,List<string> indexNames,Li
                 response = await client.BulkAsync(bulk, cancellationToken);
                 if (!response.IsValid)
                 {
+                    var errorMessage = ElasticsearchResponseHelper.GetErrorMessage(response);
+                    errorMessage = response.ItemsWithErrors.Where(item => item.Error != null).Aggregate(errorMessage, (current, item) => current + item.Error);
                     throw new ElasticsearchException(
-                        $"Bulk InsertOrUpdate Document failed at index {indexNames} :{response.ServerError.Error.Reason}");
+                        $"Bulk InsertOrUpdate Document failed at index {indexNames} :{errorMessage}");
                 }
 
                 currentIndexName = indexNames[i];
@@ -222,8 +224,10 @@ private async Task BulkAddAsync(IElasticClient client,List<string> indexNames,Li
         response = await client.BulkAsync(bulk, cancellationToken);
         if (!response.IsValid)
         {
+            var errorMessage = ElasticsearchResponseHelper.GetErrorMessage(response);
+            errorMessage = response.ItemsWithErrors.Where(item => item.Error != null).Aggregate(errorMessage, (current, item) => current + item.Error);
             throw new ElasticsearchException(
-                $"Bulk InsertOrUpdate Document failed at index {indexNames} :{response.ServerError.Error.Reason}");
+                $"Bulk InsertOrUpdate Document failed at index {indexNames} :{errorMessage}");
         }
     }
 
@@ -241,7 +245,7 @@ public async Task UpdateAsync(TEntity model, string collectionName = null,
         if (result.IsValid)
             return;
         throw new ElasticsearchException(
-            $"Update Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {result.ServerError.Error.Reason}");
+            $"Update Document failed at index {indexName} id {(model == null ? "" : model.Id.ToString())} : {ElasticsearchResponseHelper.GetErrorMessage(result)}");
     }
 
     public async Task UpdateManyAsync(List<TEntity> list, string collectionName = null,
@@ -285,7 +289,7 @@ private async Task BulkUpdateAsync(IElasticClient client, List<string> indexName
                 if (!response.IsValid)
                 {
                     throw new ElasticsearchException(
-                        $"Bulk Update Document failed at index {indexNames} :{response.ServerError.Error.Reason}");
+                        $"Bulk Update Document failed at index {indexNames} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
                 }
 
                 currentIndexName = indexNames[i];
@@ -308,7 +312,7 @@ private async Task BulkUpdateAsync(IElasticClient client, List<string> indexName
         if (!response.IsValid)
         {
             throw new ElasticsearchException(
-                $"Bulk Update Document failed at index {indexNames} :{response.ServerError.Error.Reason}");
+                $"Bulk Update Document failed at index {indexNames} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
         }
     }
 
@@ -328,7 +332,7 @@ await client.DeleteAsync(
             return;
         }
 
-        throw new ElasticsearchException($"Delete Document at index {indexName} id {id.ToString()} :{response.ServerError.Error.Reason}");
+        throw new ElasticsearchException($"Delete Document at index {indexName} id {id.ToString()} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
     }
 
     public async Task DeleteAsync(TEntity model, string collectionName = null,
@@ -349,7 +353,7 @@ await client.DeleteAsync(
         }
 
         throw new ElasticsearchException(
-            $"Delete Document at index {indexName} id {(model == null ? "" : model.Id.ToString())} :{response.ServerError.Error.Reason}");
+            $"Delete Document at index {indexName} id {(model == null ? "" : model.Id.ToString())} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
     }
 
     public async Task DeleteManyAsync(List<TEntity> list, string collectionName = null,
@@ -396,7 +400,7 @@ private async Task BulkDeleteAsync(IElasticClient client, List<string> indexName
                 if (response.ServerError != null)
                 {
                     throw new ElasticsearchException(
-                        $"Bulk Delete Document failed at index {indexNames} :{response.ServerError.Error.Reason}");
+                        $"Bulk Delete Document failed at index {indexNames} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
                 }
 
                 currentIndexName = indexNames[i];
@@ -418,7 +422,7 @@ private async Task BulkDeleteAsync(IElasticClient client, List<string> indexName
         }
 
         throw new ElasticsearchException(
-            $"Bulk Delete Document at index {indexNames} :{response.ServerError.Error.Reason}");
+            $"Bulk Delete Document at index {indexNames} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
     }
 
     public Task<IElasticClient> GetElasticsearchClientAsync(CancellationToken cancellationToken = default)
@@ -517,7 +521,7 @@ private async Task BulkAddRouteKey(IElasticClient client, List<TEntity> modelLis
         if (!response.IsValid)
         {
             throw new ElasticsearchException(
-                $"Bulk InsertOrUpdate Document failed at index {collectionRouteKeyIndexName} :{response.ServerError.Error.Reason}");
+                $"Bulk InsertOrUpdate Document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
         }
     }
 
@@ -580,7 +584,7 @@ private async Task BulkUpdateRouteKey(IElasticClient client, List<TEntity> model
         if (!response.IsValid)
         {
             throw new ElasticsearchException(
-                $"Bulk Update Document failed at index {collectionRouteKeyIndexName} :{response.ServerError.Error.Reason}");
+                $"Bulk Update Document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
         }
     }
 
@@ -627,7 +631,7 @@ private async Task BulkDeleteRouteKey(IElasticClient client, List<TEntity> model
         }
 
         throw new ElasticsearchException(
-            $"Bulk Delete Document at index {collectionRouteKeyRouteIndexName} :{response.ServerError.Error.Reason}");
+            $"Bulk Delete Document at index {collectionRouteKeyRouteIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
     }
 
 }
diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs
@@ -76,7 +76,7 @@ public async Task CreateIndexAsync(string indexName, Type type, int shard = 1, i
                         .Map(m => m.AutoMap(type)));
         if (!result.Acknowledged)
             throw new ElasticsearchException($"Create Index {indexName} failed : " +
-                                             result.ServerError.Error.Reason);
+                                             ElasticsearchResponseHelper.GetErrorMessage(result));
 
         //await client.Indices.PutAliasAsync(newName, indexName);
     }
@@ -172,7 +172,7 @@ public async Task DeleteIndexAsync(string collectionName = null, CancellationTok
                 }
 
                 // Log the error or throw an exception based on the response
-                throw new ElasticsearchException($"Failed to delete index {collectionName}: {response.ServerError.Error.Reason}");
+                throw new ElasticsearchException($"Failed to delete index {collectionName}: {ElasticsearchResponseHelper.GetErrorMessage(response)}");
             }
 
             _logger.LogInformation("Index {0} deleted successfully.", collectionName);

diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs
@@ -134,8 +134,7 @@ public async Task<List<string>> GetCollectionNameAsync(List<CollectionNameCondit
                     {
                         _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync:  result.ServerError is null result:{JsonConvert.SerializeObject(result)}");
                     }
-                    var reason = result.ServerError?.Error?.Reason ?? "Unknown error";
-                    throw new ElasticsearchException($"Search document failed at index {collectionRouteKeyIndexName} :{reason}");
+                    throw new ElasticsearchException($"Search document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(result)}");
                 }
 
                 if (result.Documents == null)
@@ -239,7 +238,7 @@ public async Task AddCollectionRouteKeyAsync(TEntity model,string fullCollection
                 {
                     throw new ElasticsearchException(
                         $"Index document failed at index {collectionRouteKeyIndexName} id {(collectionRouteKeyIndexModel == null ? "" : collectionRouteKeyIndexModel.Id)} :" +
-                        collectionRouteKeyResult.ServerError.Error.Reason);
+                        ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult));
                 }
 
             }
@@ -282,7 +281,7 @@ await GetRouteKeyCollectionAsync(collectionRouteKeyIndexId,
                     {
                         throw new ElasticsearchException(
                             $"Update document failed at index {collectionRouteKeyIndexName} id {(collectionRouteKeyIndexModel == null ? "" : collectionRouteKeyIndexModel.Id)} :" +
-                            collectionRouteKeyResult.ServerError.Error.Reason);
+                            ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult));
                     }
                 }
             }
@@ -310,7 +309,7 @@ public async Task DeleteCollectionRouteKeyAsync(string id, CancellationToken can
                 {
                     throw new ElasticsearchException(
                         $"Delete document failed at index {collectionRouteKeyIndexName} id {collectionRouteKeyIndexId} :" +
-                        collectionRouteKeyResult.ServerError.Error.Reason);
+                        ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult));
                 }
             }
         }

diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/ShardingKeyProvider.cs
@@ -1,3 +1,4 @@
+using System.Collections.Concurrent;
 using System.Linq.Expressions;
 using System.Reflection;
 using AElf.EntityMapping.Elasticsearch.Exceptions;
@@ -18,7 +19,7 @@ public class ShardingKeyProvider<TEntity> : IShardingKeyProvider<TEntity> where
     private readonly ILogger<ShardingKeyProvider<TEntity>> _logger;
 
     private List<ShardingKeyInfo<TEntity>> _shardKeyInfoList;
-    private readonly Dictionary<string, bool> _existIndexShardDictionary = new Dictionary<string, bool>();
+    private readonly ConcurrentDictionary<string, bool> _existIndexShardDictionary = new ConcurrentDictionary<string, bool>();
     private readonly Type _type = typeof(TEntity);
     private readonly string _defaultCollectionName;
     private readonly IShardingCollectionTailProvider<TEntity> _shardingCollectionTailProvider;
@@ -185,7 +186,7 @@ private async Task<bool> CheckCollectionExistAsync(string collectionName)
             if (response.ServerError != null)
             {
                 throw new ElasticsearchException(
-                    $"Exists Document failed at index {collectionName} :{response.ServerError.Error.Reason}");
+                    $"Exists Document failed at index {collectionName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}");
             }
             if (response.Exists)
             {