From eb3add5eb733f94a7cd2905174802c31e11047eb Mon Sep 17 00:00:00 2001 From: "bourne.syb" <109585311+AElfBourneShi@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:05:40 +0800 Subject: [PATCH] Feature: Remove the non-shard key related handling logic (#32) * feat: remove route key related handling logic * feat: remove collection route key attribute * feat: remove useless class * feat: remove route key code * perf: remove useless code --- .../AElfEntityMappingElasticsearchModule.cs | 1 - .../ElasticsearchCollectionNameProvider.cs | 9 +- .../IndexNameHelper.cs | 7 - .../Repositories/ElasticsearchRepository.cs | 253 +------------- .../Services/ElasticIndexService.cs | 19 -- .../Services/EnsureIndexBuildService.cs | 5 - .../Services/IElasticIndexService.cs | 2 - .../Sharding/CollectionRouteKeyProvider.cs | 322 ------------------ .../Sharding/RouteKeyCollection.cs | 20 -- .../Sharding/CollectionRouteKeyAttribute.cs | 9 - .../Sharding/CollectionRouteKeyItem.cs | 15 - .../Sharding/ICollectionRouteKeyProvider.cs | 29 -- .../Sharding/IRouteKeyCollection.cs | 10 - .../ElasticIndexServiceTests.cs | 12 - ...lasticsearchCollectionNameProviderTests.cs | 6 +- .../Entities/BlockBase.cs | 2 +- .../ElasticsearchRepositoryTests.cs | 30 +- .../CollectionRouteKeyProviderTests.cs | 140 -------- 18 files changed, 31 insertions(+), 860 deletions(-) delete mode 100644 src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs delete mode 100644 src/AElf.EntityMapping.Elasticsearch/Sharding/RouteKeyCollection.cs delete mode 100644 src/AElf.EntityMapping/Sharding/CollectionRouteKeyAttribute.cs delete mode 100644 src/AElf.EntityMapping/Sharding/CollectionRouteKeyItem.cs delete mode 100644 src/AElf.EntityMapping/Sharding/ICollectionRouteKeyProvider.cs delete mode 100644 src/AElf.EntityMapping/Sharding/IRouteKeyCollection.cs delete mode 100644 test/AElf.EntityMapping.Elasticsearch.Tests/Sharding/CollectionRouteKeyProviderTests.cs diff --git a/src/AElf.EntityMapping.Elasticsearch/AElfEntityMappingElasticsearchModule.cs b/src/AElf.EntityMapping.Elasticsearch/AElfEntityMappingElasticsearchModule.cs index dc333940..3880c68d 100644 --- a/src/AElf.EntityMapping.Elasticsearch/AElfEntityMappingElasticsearchModule.cs +++ b/src/AElf.EntityMapping.Elasticsearch/AElfEntityMappingElasticsearchModule.cs @@ -24,7 +24,6 @@ public override void ConfigureServices(ServiceConfigurationContext context) services.AddTransient(typeof(IElasticsearchRepository<,>), typeof(ElasticsearchRepository<,>)); services.AddTransient(typeof(ICollectionNameProvider<>), typeof(ElasticsearchCollectionNameProvider<>)); services.AddSingleton(typeof (IShardingKeyProvider<>), typeof (ShardingKeyProvider<>)); - services.AddSingleton(typeof(ICollectionRouteKeyProvider<>), typeof(CollectionRouteKeyProvider<>)); services.AddSingleton(typeof(IElasticsearchQueryableFactory<>), typeof(ElasticsearchQueryableFactory<>)); services.AddSingleton(typeof (IShardingCollectionTailProvider<>), typeof (ShardingCollectionTailProvider<>)); var configuration = context.Services.GetConfiguration(); diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs index 2d45ff20..3f5fa94c 100644 --- a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchCollectionNameProvider.cs @@ -12,19 +12,16 @@ public class ElasticsearchCollectionNameProvider : CollectionNameProvid { private readonly IElasticIndexService _elasticIndexService; private readonly IShardingKeyProvider _shardingKeyProvider; - private readonly ICollectionRouteKeyProvider _collectionRouteKeyProvider; private readonly AElfEntityMappingOptions _entityMappingOptions; private readonly ILogger> _logger; public ElasticsearchCollectionNameProvider(IShardingKeyProvider shardingKeyProvider, IElasticIndexService elasticIndexService, - ICollectionRouteKeyProvider collectionRouteKeyProvider, IOptions entityMappingOptions, ILogger> logger) { _elasticIndexService = elasticIndexService; _shardingKeyProvider = shardingKeyProvider; - _collectionRouteKeyProvider = collectionRouteKeyProvider; _entityMappingOptions = entityMappingOptions.Value; _logger = logger; } @@ -42,7 +39,7 @@ protected override async Task> GetCollectionNameAsync(List { GetDefaultCollectionName() }; } return shardKeyCollectionNames; @@ -71,9 +68,7 @@ protected override async Task> GetCollectionNameByEntityAsync(List< protected override async Task GetCollectionNameByIdAsync(TKey id) { - if (!_shardingKeyProvider.IsShardingCollection()) - return GetDefaultCollectionName(); - return await _collectionRouteKeyProvider.GetCollectionNameAsync(id.ToString()); + return GetDefaultCollectionName(); } protected override string FormatCollectionName(string name) diff --git a/src/AElf.EntityMapping.Elasticsearch/IndexNameHelper.cs b/src/AElf.EntityMapping.Elasticsearch/IndexNameHelper.cs index b03a96e1..314e395e 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IndexNameHelper.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IndexNameHelper.cs @@ -29,11 +29,4 @@ public static string GetDefaultFullIndexName(Type type,string collectionPrefix) : $"{collectionPrefix.ToLower()}.{GetDefaultIndexName(type)}"; return fullIndexName; } - public static string GetCollectionRouteKeyIndexName(Type type, string fieldName,string collectionPrefix) - { - var routeIndexName= collectionPrefix.IsNullOrWhiteSpace() - ? $"route.{type.Name.ToLower()}.{fieldName.ToLower()}" - : $"{collectionPrefix.ToLower()}.route.{type.Name.ToLower()}.{fieldName.ToLower()}"; - return routeIndexName; - } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs b/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs index 0ffb1c07..d5bc95f7 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Repositories/ElasticsearchRepository.cs @@ -3,7 +3,6 @@ using AElf.EntityMapping.Elasticsearch.Linq; using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Elasticsearch.Services; -using AElf.EntityMapping.Elasticsearch.Sharding; using AElf.EntityMapping.Options; using AElf.EntityMapping.Sharding; using Microsoft.Extensions.Logging; @@ -21,7 +20,6 @@ public class ElasticsearchRepository : IElasticsearchRepository _collectionNameProvider; private readonly IShardingKeyProvider _shardingKeyProvider; - private readonly ICollectionRouteKeyProvider _collectionRouteKeyProvider; private readonly IElasticIndexService _elasticIndexService; private readonly IElasticsearchQueryableFactory _elasticsearchQueryableFactory; private readonly ILogger> _logger; @@ -31,7 +29,7 @@ public ElasticsearchRepository(IElasticsearchClientProvider elasticsearchClientP IOptions aelfEntityMappingOptions, ILogger> logger, IOptions options, ICollectionNameProvider collectionNameProvider, - IShardingKeyProvider shardingKeyProvider, ICollectionRouteKeyProvider collectionRouteKeyProvider, + IShardingKeyProvider shardingKeyProvider, IElasticIndexService elasticIndexService, IElasticsearchQueryableFactory elasticsearchQueryableFactory) { _logger = logger; @@ -40,7 +38,6 @@ public ElasticsearchRepository(IElasticsearchClientProvider elasticsearchClientP _aelfEntityMappingOptions = aelfEntityMappingOptions.Value; _elasticsearchOptions = options.Value; _shardingKeyProvider = shardingKeyProvider; - _collectionRouteKeyProvider = collectionRouteKeyProvider; _elasticIndexService = elasticIndexService; _elasticsearchQueryableFactory = elasticsearchQueryableFactory; } @@ -104,8 +101,6 @@ public async Task AddAsync(TEntity model, string collectionName = null, Cancella var client = await GetElasticsearchClientAsync(cancellationToken); var result = await client.IndexAsync(model, ss => ss.Index(indexName).Refresh(_elasticsearchOptions.Refresh), cancellationToken); - - await _collectionRouteKeyProvider.AddCollectionRouteKeyAsync(model, indexName, cancellationToken); if (result.IsValid) return; @@ -126,8 +121,6 @@ public async Task AddOrUpdateAsync(TEntity model, string collectionName = null, var result = await client.UpdateAsync(DocumentPath.Id(new Id(model)), ss => ss.Index(indexName).Doc(model).RetryOnConflict(3).Refresh(_elasticsearchOptions.Refresh), cancellationToken); - - await _collectionRouteKeyProvider.UpdateCollectionRouteKeyAsync(model, cancellationToken); if (result.IsValid) return; @@ -140,8 +133,6 @@ public async Task AddOrUpdateAsync(TEntity model, string collectionName = null, await client.IndexAsync(model, ss => ss.Index(indexName).Refresh(_elasticsearchOptions.Refresh), cancellationToken); - await _collectionRouteKeyProvider.AddCollectionRouteKeyAsync(model, indexName, cancellationToken); - if (result.IsValid) return; throw new ElasticsearchException( @@ -159,34 +150,11 @@ public async Task AddOrUpdateManyAsync(List list, string collectionName _logger.LogDebug("[{1}]After GetFullCollectionNameAsync time: {0} ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"), entityName); var isSharding = _shardingKeyProvider.IsShardingCollection(); - + var client = await GetElasticsearchClientAsync(cancellationToken); - if (!isSharding) - { - await BulkAddAsync(client, indexNames, list, isSharding, cancellationToken); - return; - } - - _logger.LogDebug("[{1}]Before GetBulkAddTaskAsync time: {0} ", - DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"), - entityName); - var bulkAddTaskList = new List(); - bulkAddTaskList.Add(BulkAddAsync(client, indexNames, list, isSharding, cancellationToken)); - _logger.LogDebug("[{1}]After GetBulkAddTaskAsync time: {0} ", - DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"), entityName); - var routeKeyTaskList = - await GetBulkAddCollectionRouteKeyTasksAsync(isSharding, list, indexNames, cancellationToken); - if (routeKeyTaskList.Count > 0) - { - bulkAddTaskList.AddRange(routeKeyTaskList); - } - _logger.LogDebug("[{1}]Before Task.WhenAll time: {0} ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"), - entityName); - await Task.WhenAll(bulkAddTaskList.ToArray()); - _logger.LogDebug("[{1}]After Task.WhenAll time: {0} ", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"), - entityName); + await BulkAddAsync(client, indexNames, list, isSharding, cancellationToken); } - + private async Task BulkAddAsync(IElasticClient client,List indexNames,List list, bool isSharding, CancellationToken cancellationToken = default) { var response = new BulkResponse(); @@ -240,8 +208,6 @@ public async Task UpdateAsync(TEntity model, string collectionName = null, ss => ss.Index(indexName).Doc(model).RetryOnConflict(3).Refresh(_elasticsearchOptions.Refresh), cancellationToken); - await _collectionRouteKeyProvider.UpdateCollectionRouteKeyAsync(model, cancellationToken); - if (result.IsValid) return; throw new ElasticsearchException( @@ -254,21 +220,8 @@ public async Task UpdateManyAsync(List list, string collectionName = nu var indexNames = await GetFullCollectionNameAsync(collectionName, list); var client = await GetElasticsearchClientAsync(cancellationToken); var isSharding = _shardingKeyProvider.IsShardingCollection(); - if (!isSharding) - { - await BulkUpdateAsync(client, indexNames, list, isSharding, cancellationToken); - return; - } - - var bulkUpdateTaskList = new List(); - bulkUpdateTaskList.Add(BulkUpdateAsync(client, indexNames, list, isSharding, cancellationToken)); - var routeKeyTaskList = - await GetBulkUpdateCollectionRouteKeyTasksAsync(isSharding, list, indexNames, cancellationToken); - if (routeKeyTaskList.Count > 0) - { - bulkUpdateTaskList.AddRange(routeKeyTaskList); - } - await Task.WhenAll(bulkUpdateTaskList.ToArray()); + + await BulkUpdateAsync(client, indexNames, list, isSharding, cancellationToken); } private async Task BulkUpdateAsync(IElasticClient client, List indexNames, List list, bool isSharding, @@ -325,8 +278,6 @@ await client.DeleteAsync( new DeleteRequest(indexName, new Id(new { id = id.ToString() })) { Refresh = _elasticsearchOptions.Refresh }, cancellationToken); - await _collectionRouteKeyProvider.DeleteCollectionRouteKeyAsync(id.ToString(), cancellationToken); - if (response.ServerError == null) { return; @@ -344,9 +295,7 @@ public async Task DeleteAsync(TEntity model, string collectionName = null, await client.DeleteAsync( new DeleteRequest(indexName, new Id(model)) { Refresh = _elasticsearchOptions.Refresh }, cancellationToken); - - await _collectionRouteKeyProvider.DeleteCollectionRouteKeyAsync(model.Id.ToString(), cancellationToken); - + if (response.ServerError == null) { return; @@ -361,24 +310,10 @@ public async Task DeleteManyAsync(List list, string collectionName = nu { var indexNames = await GetFullCollectionNameAsync(collectionName, list); var isSharding = _shardingKeyProvider.IsShardingCollection(); - + var client = await GetElasticsearchClientAsync(cancellationToken); - if (!isSharding) - { - await BulkDeleteAsync(client, indexNames, list, isSharding, cancellationToken); - return; - } - - var bulkDeleteTaskList = new List(); - bulkDeleteTaskList.Add(BulkDeleteAsync(client, indexNames, list, isSharding, cancellationToken)); - var routeKeyTaskList = - await GetBulkDeleteCollectionRouteKeyTasksAsync(isSharding, list, cancellationToken); - if (routeKeyTaskList.Count > 0) - { - bulkDeleteTaskList.AddRange(routeKeyTaskList); - } - await Task.WhenAll(bulkDeleteTaskList.ToArray()); + await BulkDeleteAsync(client, indexNames, list, isSharding, cancellationToken); } private async Task BulkDeleteAsync(IElasticClient client, List indexNames, List list, bool isSharding, @@ -464,174 +399,4 @@ private async Task GetFullCollectionNameByIdAsync(TKey id, string collec ? collection : await _collectionNameProvider.GetFullCollectionNameByIdAsync(id); } - - - private async Task> GetBulkAddCollectionRouteKeyTasksAsync(bool isSharding, List modelList, - List fullCollectionNameList, CancellationToken cancellationToken = default) - { - var collectionRouteKeys = await _collectionRouteKeyProvider.GetCollectionRouteKeyItemsAsync(); - if (collectionRouteKeys != null && collectionRouteKeys.Any() && isSharding) - { - var routeKeyTaskList = new List(); - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in collectionRouteKeys) - { - routeKeyTaskList.Add(BulkAddRouteKey(client, modelList, collectionRouteKey, fullCollectionNameList, - cancellationToken)); - } - - return routeKeyTaskList; - } - - return new List(); - } - - private async Task BulkAddRouteKey(IElasticClient client, List modelList, - CollectionRouteKeyItem collectionRouteKey, List fullCollectionNameList, - CancellationToken cancellationToken) - { - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName, - _aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyBulk = new BulkRequest(collectionRouteKeyIndexName) - { - Operations = new List(), - Refresh = _elasticsearchOptions.Refresh - }; - int indexNameCount = 0; - foreach (var item in modelList) - { - // var value = item.GetType().GetProperty(collectionRouteKey.FieldName)?.GetValue(item); - var value = collectionRouteKey.GetRouteKeyValueFunc(item); - string indexName = IndexNameHelper.RemoveCollectionPrefix(fullCollectionNameList[indexNameCount], - _aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyIndexModel = new RouteKeyCollection() - { - Id = item.Id.ToString(), - CollectionName = indexName, - // SearchKey = Convert.ChangeType(value, collectionRouteKey.FieldValueType) - CollectionRouteKey = value?.ToString() - }; - collectionRouteKeyBulk.Operations.Add( - new BulkIndexOperation(collectionRouteKeyIndexModel)); - indexNameCount++; - } - - var response = await client.BulkAsync(collectionRouteKeyBulk, cancellationToken); - if (!response.IsValid) - { - throw new ElasticsearchException( - $"Bulk InsertOrUpdate Document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}"); - } - } - - private async Task> GetBulkUpdateCollectionRouteKeyTasksAsync(bool isSharding, List modelList, - List fullCollectionNameList, CancellationToken cancellationToken = default) - { - var collectionRouteKeys = await _collectionRouteKeyProvider.GetCollectionRouteKeyItemsAsync(); - if (collectionRouteKeys != null && collectionRouteKeys.Any() && isSharding) - { - var routeKeyTaskList = new List(); - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in collectionRouteKeys) - { - routeKeyTaskList.Add(BulkUpdateRouteKey(client, modelList, collectionRouteKey, fullCollectionNameList, - cancellationToken)); - } - - return routeKeyTaskList; - } - - return new List(); - } - - private async Task BulkUpdateRouteKey(IElasticClient client, List modelList, - CollectionRouteKeyItem collectionRouteKey, List fullCollectionNameList, - CancellationToken cancellationToken) - { - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName, - _aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyBulk = new BulkRequest(collectionRouteKeyIndexName) - { - Operations = new List(), - Refresh = _elasticsearchOptions.Refresh - }; - int indexNameCount = 0; - foreach (var item in modelList) - { - // var value = item.GetType().GetProperty(collectionRouteKey.FieldName)?.GetValue(item); - var value = collectionRouteKey.GetRouteKeyValueFunc(item); - string indexName = IndexNameHelper.RemoveCollectionPrefix(fullCollectionNameList[indexNameCount], - _aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyIndexModel = new RouteKeyCollection() - { - Id = item.Id.ToString(), - CollectionName = indexName, - // SearchKey = Convert.ChangeType(value, collectionRouteKey.FieldValueType) - CollectionRouteKey = value?.ToString() - }; - var updateOperation = new BulkUpdateOperation(new Id(collectionRouteKeyIndexModel)) - { - Doc = collectionRouteKeyIndexModel, - Index = collectionRouteKeyIndexName - }; - collectionRouteKeyBulk.Operations.Add(updateOperation); - indexNameCount++; - } - - var response = await client.BulkAsync(collectionRouteKeyBulk, cancellationToken); - if (!response.IsValid) - { - throw new ElasticsearchException( - $"Bulk Update Document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}"); - } - } - - private async Task> GetBulkDeleteCollectionRouteKeyTasksAsync(bool isSharding, List modelList, - CancellationToken cancellationToken = default) - { - var collectionRouteKeys = await _collectionRouteKeyProvider.GetCollectionRouteKeyItemsAsync(); - if (collectionRouteKeys != null && collectionRouteKeys.Any() && isSharding) - { - var routeKeyTaskList = new List(); - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in collectionRouteKeys) - { - routeKeyTaskList.Add(BulkDeleteRouteKey(client, modelList, collectionRouteKey, cancellationToken)); - } - - return routeKeyTaskList; - } - - return new List(); - } - - private async Task BulkDeleteRouteKey(IElasticClient client, List modelList, - CollectionRouteKeyItem collectionRouteKey, CancellationToken cancellationToken) - { - var collectionRouteKeyRouteIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName, - _aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyRouteBulk = new BulkRequest(collectionRouteKeyRouteIndexName) - { - Operations = new List(), - Refresh = _elasticsearchOptions.Refresh - }; - foreach (var item in modelList) - { - collectionRouteKeyRouteBulk.Operations.Add(new BulkDeleteOperation(new Id(item))); - } - - var response = await client.BulkAsync(collectionRouteKeyRouteBulk, cancellationToken); - - if (response.ServerError == null) - { - return; - } - - throw new ElasticsearchException( - $"Bulk Delete Document at index {collectionRouteKeyRouteIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(response)}"); - } - } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs index 4f9d2a4c..27419f55 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/ElasticIndexService.cs @@ -127,25 +127,6 @@ public async Task CreateIndexTemplateAsync(string indexTemplateName,string index _logger.LogInformation("Index template {indexTemplateName} created successfully", indexTemplateName); } } - - public async Task CreateCollectionRouteKeyIndexAsync(Type type, int shard = 1, int numberOfReplicas = 1) - { - var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - foreach (var property in properties) - { - CollectionRouteKeyAttribute shardRouteAttribute = (CollectionRouteKeyAttribute)Attribute.GetCustomAttribute(property, typeof(CollectionRouteKeyAttribute)); - if (shardRouteAttribute != null) - { - if (property.PropertyType != typeof(string)) - { - throw new NotSupportedException( - $"{type.Name} Attribute Error! NeedShardRouteAttribute only support string type, please check field: {property.Name}"); - } - var indexName = IndexNameHelper.GetCollectionRouteKeyIndexName(type, property.Name,_entityMappingOptions.CollectionPrefix); - await CreateIndexAsync(indexName, typeof(RouteKeyCollection), shard, numberOfReplicas); - } - } - } public async Task DeleteIndexAsync(string collectionName = null, CancellationToken cancellationToken = default) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs index 5bfdf065..8859f7b5 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/EnsureIndexBuildService.cs @@ -54,11 +54,6 @@ private async Task HandleModuleAsync(Type moduleType) await _elasticIndexService.CreateIndexTemplateAsync(indexTemplateName, indexName, t, _elasticsearchOptions.NumberOfShards, _elasticsearchOptions.NumberOfReplicas); - //create index marked field cache - // await _elasticIndexService.InitializeCollectionRouteKeyCacheAsync(t); - //create non shard key route index - await _elasticIndexService.CreateCollectionRouteKeyIndexAsync(t, _elasticsearchOptions.NumberOfShards, - _elasticsearchOptions.NumberOfReplicas); await CreateShardingCollectionTailIndexAsync(); } else diff --git a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs index b4b9f702..6a081db1 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Services/IElasticIndexService.cs @@ -8,7 +8,5 @@ Task CreateIndexAsync(string indexName, Type indexEntityType, int shard = 1, int Task CreateIndexTemplateAsync(string indexTemplateName, string indexName, Type indexEntityType, int numberOfShards, int numberOfReplicas); - Task CreateCollectionRouteKeyIndexAsync(Type indexEntityType, int numberOfShards, - int numberOfReplicas); Task DeleteIndexAsync(string collectionName = null, CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs deleted file mode 100644 index 3e535d36..00000000 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/CollectionRouteKeyProvider.cs +++ /dev/null @@ -1,322 +0,0 @@ -using System.Linq.Expressions; -using System.Reflection; -using AElf.EntityMapping.Elasticsearch.Exceptions; -using AElf.EntityMapping.Elasticsearch.Options; -using AElf.EntityMapping.Elasticsearch.Services; -using AElf.EntityMapping.Options; -using AElf.EntityMapping.Sharding; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Nest; -using Newtonsoft.Json; -using Volo.Abp.DependencyInjection; -using Volo.Abp.Domain.Entities; - -namespace AElf.EntityMapping.Elasticsearch.Sharding; - -public class CollectionRouteKeyProvider:ICollectionRouteKeyProvider where TEntity : class, IEntity -{ - private readonly IAbpLazyServiceProvider _lazyServiceProvider; - // private IElasticsearchRepository _collectionRouteKeyIndexRepository => LazyServiceProvider - // .LazyGetRequiredService>(); - private readonly IElasticIndexService _elasticIndexService; - private readonly IShardingKeyProvider _shardingKeyProvider; - private List> _collectionRouteKeys; - private readonly IElasticsearchClientProvider _elasticsearchClientProvider; - private readonly AElfEntityMappingOptions _aelfEntityMappingOptions; - private readonly ElasticsearchOptions _elasticsearchOptions; - private readonly ILogger> _logger; - - public CollectionRouteKeyProvider(IElasticsearchClientProvider elasticsearchClientProvider, - IShardingKeyProvider shardingKeyProvider, - IAbpLazyServiceProvider lazyServiceProvider, - IOptions aelfEntityMappingOptions, - IOptions elasticsearchOptions, - ILogger> logger, - IElasticIndexService elasticIndexService) - { - _elasticIndexService = elasticIndexService; - _elasticsearchClientProvider = elasticsearchClientProvider; - _shardingKeyProvider = shardingKeyProvider; - _lazyServiceProvider= lazyServiceProvider; - _aelfEntityMappingOptions = aelfEntityMappingOptions.Value; - _elasticsearchOptions = elasticsearchOptions.Value; - _logger = logger; - - InitializeCollectionRouteKeys(); - } - - private void InitializeCollectionRouteKeys() - { - if (_collectionRouteKeys == null) - { - _collectionRouteKeys = new List>(); - Type type = typeof(TEntity); - var properties = type.GetProperties(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); - foreach (var property in properties) - { - var collectionRouteKeyItem = new CollectionRouteKeyItem() - { - FieldName = property.Name, - CollectionName = type.Name - }; - //Find the field with the CollectionRouteKeyAttribute annotation set - CollectionRouteKeyAttribute routeKeyAttribute = (CollectionRouteKeyAttribute)Attribute.GetCustomAttribute(property, typeof(CollectionRouteKeyAttribute)); - if (routeKeyAttribute != null) - { - // Creates a Func expression that gets the value of the property - var parameter = Expression.Parameter(type, "entity"); - var propertyAccess = Expression.Property(parameter, property); - var getPropertyFunc = Expression.Lambda>(propertyAccess, parameter).Compile(); - collectionRouteKeyItem.GetRouteKeyValueFunc = getPropertyFunc; - _collectionRouteKeys.Add(collectionRouteKeyItem); - } - } - // _logger.LogDebug($"CollectionRouteKeyProvider.InitializeCollectionRouteKeys: _collectionRouteKeys: {JsonConvert.SerializeObject(_collectionRouteKeys.Select(n=>n.FieldName).ToList())}"); - - } - } - - public async Task> GetCollectionNameAsync(List conditions) - { - var collectionNameList = new List(); - if (_collectionRouteKeys == null || _collectionRouteKeys.Count == 0) - { - return collectionNameList; - } - - foreach (var condition in conditions) - { - var collectionRouteKey = _collectionRouteKeys.FirstOrDefault(f => f.FieldName == condition.Key); - if (collectionRouteKey == null) - { - continue; - } - - // _logger.LogDebug($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: " + - // $"collectionRouteKey: {JsonConvert.SerializeObject(collectionRouteKey.FieldName)}"); - - if (condition.Value == null) - { - continue; - } - - // var fieldValue = Convert.ChangeType(condition.Value, collectionRouteKey.FieldValueType); - var fieldValue = condition.Value.ToString(); - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName, - _aelfEntityMappingOptions.CollectionPrefix); - _logger.LogDebug($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: " + - $"collectionRouteKeyIndexName: {collectionRouteKeyIndexName}"); - if (condition.Type == ConditionType.Equal) - { - if (_elasticsearchClientProvider == null) - { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: elasticsearchClientProvider is null"); - } - var client = _elasticsearchClientProvider.GetClient(); - if (client == null) - { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: client is null"); - } - var result = await client.SearchAsync(s => - s.Index(collectionRouteKeyIndexName) - .Query(q => q.Term(t => t.Field(f => f.CollectionRouteKey).Value(fieldValue))) - .Collapse(c => c.Field(f => f.CollectionName)) - .Size(1000)); - if (result == null) - { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result is null fieldValue:{fieldValue}"); - } - if (!result.IsValid) - { - if (result.ServerError == null || result.ServerError.Error == null || result.ServerError.Error.Reason == null) - { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result.ServerError is null result:{JsonConvert.SerializeObject(result)}"); - } - throw new ElasticsearchException($"Search document failed at index {collectionRouteKeyIndexName} :{ElasticsearchResponseHelper.GetErrorMessage(result)}"); - } - - if (result.Documents == null) - { - _logger.LogError($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: result.Documents is null fieldValue:{fieldValue}"); - } - var collectionList = result.Documents.ToList(); - _logger.LogDebug($"CollectionRouteKeyProvider.GetShardCollectionNameListByConditionsAsync: " + - $"collectionList: {JsonConvert.SerializeObject(collectionList)}"); - var nameList = collectionList.Select(x => x.CollectionName).Distinct().ToList(); - if (collectionNameList.Count == 0) - { - collectionNameList.AddRange(nameList); - } - else - { - collectionNameList = collectionNameList.Intersect(nameList).ToList(); - } - } - } - - return collectionNameList; - } - - public async Task GetCollectionNameAsync(string id) - { - var collectionName=string.Empty; - if (_collectionRouteKeys == null || _collectionRouteKeys.Count == 0) - { - return collectionName; - } - - var collectionRouteKey= _collectionRouteKeys[0]; - var collectionRouteKeyIndexName = IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName,_aelfEntityMappingOptions.CollectionPrefix); - // var routeIndex=await _collectionRouteKeyIndexRepository.GetAsync(id, collectionRouteKeyIndexName); - var routeIndex = await GetRouteKeyCollectionAsync(id, collectionRouteKeyIndexName); - if (routeIndex != null) - { - collectionName = routeIndex.CollectionName; - } - - return collectionName; - } - - public Task>> GetCollectionRouteKeyItemsAsync() - { - return Task.FromResult(_collectionRouteKeys); - } - - public async Task GetRouteKeyCollectionAsync(string id, string indexName, CancellationToken cancellationToken = default) - { - var client = _elasticsearchClientProvider.GetClient(); - var selector = new Func, IGetRequest>(s => s - .Index(indexName)); - var result = new GetResponse(); - try - { - result = await client.GetAsync(new Nest.DocumentPath(new Id(new { id = id.ToString() })), - selector, cancellationToken); - } - catch (Exception e) - { - throw new ElasticsearchException($"Get Document failed at index {indexName} id {id.ToString()}", e); - } - return result.Found ? result.Source : null; - } - - - - public async Task AddCollectionRouteKeyAsync(TEntity model,string fullCollectionName,CancellationToken cancellationToken = default) - { - if (!_shardingKeyProvider.IsShardingCollection()) - { - return; - } - - string indexName = - IndexNameHelper.RemoveCollectionPrefix(fullCollectionName, _aelfEntityMappingOptions.CollectionPrefix); - - if (_collectionRouteKeys!=null && _collectionRouteKeys.Any()) - { - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in _collectionRouteKeys) - { - // var value = model.GetType().GetProperty(collectionRouteKey.FieldName)?.GetValue(model); - var value = collectionRouteKey.GetRouteKeyValueFunc(model); - var collectionRouteKeyIndexModel = new RouteKeyCollection() - { - Id = model.Id.ToString(), - CollectionName = indexName, - // SearchKey = Convert.ChangeType(value, collectionRouteKey.FieldValueType) - CollectionRouteKey = value?.ToString() - }; - - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName,_aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyResult = await client.IndexAsync(collectionRouteKeyIndexModel, - ss => ss.Index(collectionRouteKeyIndexName).Refresh(_elasticsearchOptions.Refresh), - cancellationToken); - if (!collectionRouteKeyResult.IsValid) - { - throw new ElasticsearchException( - $"Index document failed at index {collectionRouteKeyIndexName} id {(collectionRouteKeyIndexModel == null ? "" : collectionRouteKeyIndexModel.Id)} :" + - ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult)); - } - - } - } - } - - public async Task UpdateCollectionRouteKeyAsync(TEntity model, CancellationToken cancellationToken = default) - { - if (!_shardingKeyProvider.IsShardingCollection()) - { - return; - } - - if (_collectionRouteKeys!=null && _collectionRouteKeys.Any()) - { - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in _collectionRouteKeys) - { - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName,_aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyIndexId = model.Id.ToString(); - var collectionRouteKeyIndexModel = - await GetRouteKeyCollectionAsync(collectionRouteKeyIndexId, - collectionRouteKeyIndexName, cancellationToken); - // var collectionRouteKeyIndexModel = GetAsync((TKey)Convert.ChangeType(collectionRouteKeyIndexId, typeof(TKey)), collectionRouteKeyIndexName) as RouteKeyCollection; - - // var value = model.GetType().GetProperty(collectionRouteKey.FieldName)?.GetValue(model); - var value = collectionRouteKey.GetRouteKeyValueFunc(model); - if (collectionRouteKeyIndexModel != null && collectionRouteKeyIndexModel.CollectionRouteKey != value?.ToString()) - { - // collectionRouteKeyIndexModel.SearchKey = Convert.ChangeType(value, collectionRouteKey.FieldValueType); - collectionRouteKeyIndexModel.CollectionRouteKey = value?.ToString(); - - var collectionRouteKeyResult = await client.UpdateAsync( - DocumentPath.Id(new Id(collectionRouteKeyIndexModel)), - ss => ss.Index(collectionRouteKeyIndexName).Doc((RouteKeyCollection)collectionRouteKeyIndexModel).RetryOnConflict(3) - .Refresh(_elasticsearchOptions.Refresh), - cancellationToken); - if (!collectionRouteKeyResult.IsValid) - { - throw new ElasticsearchException( - $"Update document failed at index {collectionRouteKeyIndexName} id {(collectionRouteKeyIndexModel == null ? "" : collectionRouteKeyIndexModel.Id)} :" + - ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult)); - } - } - } - } - } - - public async Task DeleteCollectionRouteKeyAsync(string id, CancellationToken cancellationToken = default) - { - if (!_shardingKeyProvider.IsShardingCollection()) - { - return; - } - if (_collectionRouteKeys!=null && _collectionRouteKeys.Any()) - { - var client = await GetElasticsearchClientAsync(cancellationToken); - foreach (var collectionRouteKey in _collectionRouteKeys) - { - var collectionRouteKeyIndexName = - IndexNameHelper.GetCollectionRouteKeyIndexName(typeof(TEntity), collectionRouteKey.FieldName,_aelfEntityMappingOptions.CollectionPrefix); - var collectionRouteKeyIndexId = id; - var collectionRouteKeyResult=await client.DeleteAsync( - new DeleteRequest(collectionRouteKeyIndexName, new Id(new { id = collectionRouteKeyIndexId.ToString() })) - { Refresh = _elasticsearchOptions.Refresh }, cancellationToken); - if (collectionRouteKeyResult.ServerError != null) - { - throw new ElasticsearchException( - $"Delete document failed at index {collectionRouteKeyIndexName} id {collectionRouteKeyIndexId} :" + - ElasticsearchResponseHelper.GetErrorMessage(collectionRouteKeyResult)); - } - } - } - } - - private Task GetElasticsearchClientAsync(CancellationToken cancellationToken = default) - { - return Task.FromResult(_elasticsearchClientProvider.GetClient()); - } -} \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Sharding/RouteKeyCollection.cs b/src/AElf.EntityMapping.Elasticsearch/Sharding/RouteKeyCollection.cs deleted file mode 100644 index 6faf43c5..00000000 --- a/src/AElf.EntityMapping.Elasticsearch/Sharding/RouteKeyCollection.cs +++ /dev/null @@ -1,20 +0,0 @@ -using AElf.EntityMapping.Entities; -using AElf.EntityMapping.Sharding; -using Nest; -using Volo.Abp.Domain.Entities; - -namespace AElf.EntityMapping.Elasticsearch.Sharding; - -public class RouteKeyCollection:Entity,IEntity,IEntityMappingEntity,IRouteKeyCollection -{ - [Keyword]public string Id { get; set; } - - [Keyword]public string CollectionName { get; set; } - //can only support string type - [Keyword]public string CollectionRouteKey { get; set; } - - public override object[] GetKeys() - { - return new object[] {Id}; - } -} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Sharding/CollectionRouteKeyAttribute.cs b/src/AElf.EntityMapping/Sharding/CollectionRouteKeyAttribute.cs deleted file mode 100644 index 961a9b65..00000000 --- a/src/AElf.EntityMapping/Sharding/CollectionRouteKeyAttribute.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace AElf.EntityMapping.Sharding; - -[AttributeUsage(AttributeTargets.Property|AttributeTargets.Field, AllowMultiple = false, Inherited = false)] -public class CollectionRouteKeyAttribute : Attribute -{ - public CollectionRouteKeyAttribute() - { - } -} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Sharding/CollectionRouteKeyItem.cs b/src/AElf.EntityMapping/Sharding/CollectionRouteKeyItem.cs deleted file mode 100644 index a6f53277..00000000 --- a/src/AElf.EntityMapping/Sharding/CollectionRouteKeyItem.cs +++ /dev/null @@ -1,15 +0,0 @@ -using AElf.EntityMapping.Entities; -using Volo.Abp.Domain.Entities; - -namespace AElf.EntityMapping.Sharding; - -public class CollectionRouteKeyItem -{ - public string FieldName { get; set; } - // public Type FieldValueType { get; set; } - // public string FieldValueType { get; set; } - public string CollectionName { get; set; } - // public bool IsShardKey { get; set; } - // public bool IsRouteKey { get; set; } - public Func GetRouteKeyValueFunc { get; set; } -} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Sharding/ICollectionRouteKeyProvider.cs b/src/AElf.EntityMapping/Sharding/ICollectionRouteKeyProvider.cs deleted file mode 100644 index 54246b16..00000000 --- a/src/AElf.EntityMapping/Sharding/ICollectionRouteKeyProvider.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace AElf.EntityMapping.Sharding; - -public interface ICollectionRouteKeyProvider where TEntity : class -{ - /// - /// - /// - /// - /// ElasticSearch Index Entity - /// - Task> GetCollectionNameAsync(List conditions); - - Task GetCollectionNameAsync(string id); - - Task>> GetCollectionRouteKeyItemsAsync(); - - Task GetRouteKeyCollectionAsync(string id, string collectionName, CancellationToken cancellationToken = default); - - // Task>> AddManyCollectionRouteKeyAsync(List modelList, List fullCollectionNameList, CancellationToken cancellationToken = default); - - Task AddCollectionRouteKeyAsync(TEntity model, string fullCollectionName, CancellationToken cancellationToken = default); - - Task UpdateCollectionRouteKeyAsync(TEntity model, CancellationToken cancellationToken = default); - - // Task>> DeleteManyCollectionRouteKeyAsync(List modelList, CancellationToken cancellationToken = default); - - Task DeleteCollectionRouteKeyAsync(string id, CancellationToken cancellationToken = default); - -} \ No newline at end of file diff --git a/src/AElf.EntityMapping/Sharding/IRouteKeyCollection.cs b/src/AElf.EntityMapping/Sharding/IRouteKeyCollection.cs deleted file mode 100644 index dafafe5c..00000000 --- a/src/AElf.EntityMapping/Sharding/IRouteKeyCollection.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace AElf.EntityMapping.Sharding; - -public interface IRouteKeyCollection -{ - string Id { get; set; } - - string CollectionName { get; set; } - //can only support string type - string CollectionRouteKey { get; set; } -} \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticIndexServiceTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticIndexServiceTests.cs index 208cea18..8b8e9719 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticIndexServiceTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticIndexServiceTests.cs @@ -10,21 +10,9 @@ namespace AElf.EntityMapping.Elasticsearch; public class ElasticIndexServiceTests: AElfElasticsearchTestBase { private readonly IElasticIndexService _elasticIndexService; - private readonly IDistributedCache>> _indexMarkFieldCache; public ElasticIndexServiceTests() { _elasticIndexService = GetRequiredService(); - _indexMarkFieldCache= GetRequiredService>>>(); } - - // [Fact] - // public async Task InitializeIndexMarkedField_Test() - // { - // await _elasticIndexService.InitializeCollectionRouteKeyCacheAsync(typeof(BlockIndex)); - // var cacheName = _elasticIndexService.GetCollectionRouteKeyCacheName(typeof(BlockIndex)); - // var collectionMarkFieldList = await _indexMarkFieldCache.GetAsync(cacheName); - // collectionMarkFieldList.ShouldNotBeNull(); - // collectionMarkFieldList.Count.ShouldBeGreaterThan(1); - // } } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs index fcbc73ef..d67131f0 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/ElasticsearchCollectionNameProviderTests.cs @@ -150,10 +150,12 @@ public async Task GetFullCollectionNameById_Test() await InitBlocksAsync(); var collectionNames = await _collectionNameProvider.GetFullCollectionNameByIdAsync("block1"); - collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-0"); + // collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-0"); + collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex"); collectionNames = await _collectionNameProvider.GetFullCollectionNameByIdAsync("block6"); - collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); + // collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); + collectionNames.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex"); } private async Task InitBlocksAsync() diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs index 6f36bb3e..a450d9bc 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Entities/BlockBase.cs @@ -11,7 +11,7 @@ public class BlockBase : AElfIndexerEntity, IBlockchainData [ShardPropertyAttributes("ChainId", 1)] public string ChainId { get; set; } - [CollectionRouteKey] [Keyword] public string BlockHash { get; set; } + [Keyword] public string BlockHash { get; set; } [ShardPropertyAttributes("BlockHeight", 3)] public long BlockHeight { get; set; } diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index 90cbe5e9..bee149dc 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -414,21 +414,21 @@ public async Task Get_Test() block.LogEventCount.ShouldBe(blockIndex.LogEventCount); block.ChainId.ShouldBe(blockIndex.ChainId); - for (int i = 1; i <= 7; i++) - { - await _elasticsearchRepository.AddAsync(new BlockIndex - { - Id = "block" + i, - BlockHash = "BlockHash" + i, - BlockHeight = i, - BlockTime = DateTime.Now.AddDays(-10 + i), - LogEventCount = i, - ChainId = "AELF" - }); - } - - block = await _elasticsearchRepository.GetAsync("block7"); - block.Id.ShouldBe("block7"); + // for (int i = 1; i <= 7; i++) + // { + // await _elasticsearchRepository.AddAsync(new BlockIndex + // { + // Id = "block" + i, + // BlockHash = "BlockHash" + i, + // BlockHeight = i, + // BlockTime = DateTime.Now.AddDays(-10 + i), + // LogEventCount = i, + // ChainId = "AELF" + // }); + // } + // + // block = await _elasticsearchRepository.GetAsync("block7"); + // block.Id.ShouldBe("block7"); } [Fact] diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Sharding/CollectionRouteKeyProviderTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Sharding/CollectionRouteKeyProviderTests.cs deleted file mode 100644 index eeb07758..00000000 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Sharding/CollectionRouteKeyProviderTests.cs +++ /dev/null @@ -1,140 +0,0 @@ -using AElf.EntityMapping.Elasticsearch.Entities; -using AElf.EntityMapping.Elasticsearch.Repositories; -using AElf.EntityMapping.Options; -using AElf.EntityMapping.Sharding; -using Microsoft.Extensions.Options; -using Shouldly; -using Xunit; - -namespace AElf.EntityMapping.Elasticsearch.Sharding; - -public class CollectionRouteKeyProviderTests: AElfElasticsearchTestBase -{ - private readonly ICollectionRouteKeyProvider _blockIndexCollectionRouteKeyProvider; - private readonly IElasticsearchRepository _elasticsearchRepository; - private readonly AElfEntityMappingOptions _option; - - public CollectionRouteKeyProviderTests() - { - _blockIndexCollectionRouteKeyProvider = GetRequiredService>(); - _elasticsearchRepository = GetRequiredService>(); - _option = GetRequiredService>().Value; - } - - [Fact] - public async Task GetRouteKeys_Test() - { - List> routeKeys = await _blockIndexCollectionRouteKeyProvider.GetCollectionRouteKeyItemsAsync(); - - routeKeys.Count.ShouldBe(1); - routeKeys[0].FieldName.ShouldBe(nameof(BlockIndex.BlockHash)); - // routeKeys[0].FieldValueType.ShouldBe(typeof(string).ToString()); - // routeKeys[0].IsRouteKey.ShouldBeTrue(); - } - - [Fact] - public async Task GetShardCollectionNameListByConditions_Test() - { - await InitBlocksAsync(); - - var collectionNameCondition = new List(); - var indexes = - await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync( - collectionNameCondition); - indexes.Count.ShouldBe(0); - - collectionNameCondition.Add(new CollectionNameCondition - { - Key = nameof(BlockIndex.BlockHash), - Value = "BlockHash7", - Type = ConditionType.Equal - }); - indexes = - await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync( - collectionNameCondition); - indexes.Count.ShouldBe(1); - // indexes[0].ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); - indexes[0].ShouldBe($"blockindex-aelf-1"); - - collectionNameCondition.Add(new CollectionNameCondition - { - Key = nameof(BlockIndex.BlockHash), - Value = "BlockHash6", - Type = ConditionType.Equal - }); - indexes = - await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync( - collectionNameCondition); - indexes.Count.ShouldBe(1); - // indexes[0].ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); - indexes[0].ShouldBe($"blockindex-aelf-1"); - - collectionNameCondition.Add(new CollectionNameCondition - { - Key = nameof(BlockIndex.BlockHash), - Value = "BlockHash1", - Type = ConditionType.Equal - }); - indexes = - await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync( - collectionNameCondition); - indexes.Count.ShouldBe(0); - } - - [Fact] - public async Task GetShardCollectionNameById_Test() - { - await InitBlocksAsync(); - - var index = await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync("block1"); - // index.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-0"); - index.ShouldBe($"blockindex-aelf-0"); - - index = await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync("block2"); - // index.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-0"); - index.ShouldBe($"blockindex-aelf-0"); - - index = await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync("block7"); - // index.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); - index.ShouldBe($"blockindex-aelf-1"); - - index = await _blockIndexCollectionRouteKeyProvider.GetCollectionNameAsync("block8"); - index.ShouldBeEmpty(); - } - - [Fact] - public async Task GetCollectionRouteKeyIndex_Test() - { - var routeIndex = $"{_option.CollectionPrefix.ToLower()}.route.blockindex.blockhash"; - await InitBlocksAsync(); - - var route = await _blockIndexCollectionRouteKeyProvider.GetRouteKeyCollectionAsync("block1", routeIndex); - route.Id.ShouldBe("block1"); - route.CollectionRouteKey.ShouldBe("BlockHash1"); - // route.ShardCollectionName.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-0"); - route.CollectionName.ShouldBe($"blockindex-aelf-0"); - - route = await _blockIndexCollectionRouteKeyProvider.GetRouteKeyCollectionAsync("block6", routeIndex); - route.Id.ShouldBe("block6"); - route.CollectionRouteKey.ShouldBe("BlockHash6"); - // route.ShardCollectionName.ShouldBe($"{_option.CollectionPrefix.ToLower()}.blockindex-aelf-1"); - route.CollectionName.ShouldBe($"blockindex-aelf-1"); - } - - private async Task InitBlocksAsync() - { - for (int i = 1; i <= 7; i++) - { - var blockIndex = new BlockIndex - { - Id = "block" + i, - BlockHash = "BlockHash" + i, - BlockHeight = i, - BlockTime = DateTime.Now.AddDays(-10 + i), - LogEventCount = i, - ChainId = "AELF" - }; - await _elasticsearchRepository.AddAsync(blockIndex); - } - } -} \ No newline at end of file