diff --git a/src/Raven.Client/Documents/Operations/ETL/Queue/AzureQueueStorageConnectionSettings.cs b/src/Raven.Client/Documents/Operations/ETL/Queue/AzureQueueStorageConnectionSettings.cs index 64be25aa693f..e41e66030e12 100644 --- a/src/Raven.Client/Documents/Operations/ETL/Queue/AzureQueueStorageConnectionSettings.cs +++ b/src/Raven.Client/Documents/Operations/ETL/Queue/AzureQueueStorageConnectionSettings.cs @@ -6,7 +6,11 @@ namespace Raven.Client.Documents.Operations.ETL.Queue; public sealed class AzureQueueStorageConnectionSettings { - public Authentication Authentication; + public EntraId EntraId { get; set; } + + public string ConnectionString { get; set; } + + public bool Passwordless { get; set; } public string GetStorageUrl() { @@ -18,22 +22,22 @@ public string GetStorageAccountName() { string storageAccountName = ""; - if (Authentication.ConnectionString != null) + if (ConnectionString != null) { - var accountNamePart = Authentication.ConnectionString.Split(';') + var accountNamePart = ConnectionString.Split(';') .FirstOrDefault(part => part.StartsWith("AccountName=", StringComparison.OrdinalIgnoreCase)); if (accountNamePart == null) { throw new ArgumentException("Storage account name not found in the connection string.", - nameof(Authentication.ConnectionString)); + nameof(ConnectionString)); } storageAccountName = accountNamePart.Substring("AccountName=".Length); } - else if (Authentication.EntraId != null) + else if (EntraId != null) { - storageAccountName = Authentication.EntraId.StorageAccountName; + storageAccountName = EntraId.StorageAccountName; } return storageAccountName; @@ -43,40 +47,24 @@ public DynamicJsonValue ToJson() { var json = new DynamicJsonValue { - [nameof(Authentication)] = Authentication == null + [nameof(ConnectionString)] = ConnectionString, + [nameof(Passwordless)] = Passwordless, + [nameof(EntraId)] = EntraId == null ? null : new DynamicJsonValue { - [nameof(Authentication.ConnectionString)] = Authentication.ConnectionString, - [nameof(Authentication.Passwordless)] = Authentication.Passwordless, - [nameof(Authentication.EntraId)] = - Authentication.EntraId == null - ? null - : new DynamicJsonValue - { - [nameof(Authentication.EntraId.StorageAccountName)] = - Authentication?.EntraId?.StorageAccountName, - [nameof(Authentication.EntraId.TenantId)] = - Authentication?.EntraId?.TenantId, - [nameof(Authentication.EntraId.ClientId)] = - Authentication?.EntraId?.ClientId, - [nameof(Authentication.EntraId.ClientSecret)] = - Authentication?.EntraId?.ClientSecret - } + [nameof(EntraId.StorageAccountName)] = EntraId?.StorageAccountName, + [nameof(EntraId.TenantId)] = EntraId?.TenantId, + [nameof(EntraId.ClientId)] = EntraId?.ClientId, + [nameof(EntraId.ClientSecret)] = EntraId?.ClientSecret } }; + return json; } } -public sealed class Authentication -{ - public EntraId EntraId { get; set; } - public string ConnectionString { get; set; } - public bool Passwordless { get; set; } -} - public sealed class EntraId { public string StorageAccountName { get; set; } diff --git a/src/Raven.Client/Documents/Operations/ETL/Queue/QueueConnectionString.cs b/src/Raven.Client/Documents/Operations/ETL/Queue/QueueConnectionString.cs index 93f4a33ac51e..62c2b8a52352 100644 --- a/src/Raven.Client/Documents/Operations/ETL/Queue/QueueConnectionString.cs +++ b/src/Raven.Client/Documents/Operations/ETL/Queue/QueueConnectionString.cs @@ -34,7 +34,8 @@ protected override void ValidateImpl(ref List errors) } break; case QueueBrokerType.AzureQueueStorage: - if (AzureQueueStorageConnectionSettings?.Authentication == null) + //todo djordje: better validation, at least one auth method + if (AzureQueueStorageConnectionSettings == null) { errors.Add($"{nameof(AzureQueueStorageConnectionSettings)} has no valid setting."); } diff --git a/src/Raven.Server/Commercial/LicenseStatus.cs b/src/Raven.Server/Commercial/LicenseStatus.cs index 653181381f89..876ad8c533a5 100644 --- a/src/Raven.Server/Commercial/LicenseStatus.cs +++ b/src/Raven.Server/Commercial/LicenseStatus.cs @@ -216,9 +216,7 @@ public bool CanAutoRenewLetsEncryptCertificate public bool HasElasticSearchEtl => Enabled(LicenseAttribute.ElasticSearchEtl); - //public bool HasQueueEtl => Enabled(LicenseAttribute.QueueEtl); - public bool HasQueueEtl => true; - + public bool HasQueueEtl => Enabled(LicenseAttribute.QueueEtl); public bool HasPowerBI => Enabled(LicenseAttribute.PowerBI); diff --git a/src/Raven.Server/Config/Categories/EtlConfiguration.cs b/src/Raven.Server/Config/Categories/EtlConfiguration.cs index bec32b3946de..e22850144bab 100644 --- a/src/Raven.Server/Config/Categories/EtlConfiguration.cs +++ b/src/Raven.Server/Config/Categories/EtlConfiguration.cs @@ -54,7 +54,7 @@ public sealed class EtlConfiguration : ConfigurationCategory public TimeSetting KafkaInitTransactionsTimeout { get; set; } [Description("Lifespan of a message in the queue")] - [DefaultValue(604800)] + [DefaultValue(604800)] // 7 days (Azure default) [TimeUnit(TimeUnit.Seconds)] [ConfigurationEntry("ETL.Queue.AzureQueueStorage.TimeToLiveInSec", ConfigurationEntryScope.ServerWideOrPerDatabase)] public TimeSetting AzureQueueStorageTimeToLive{ get; set; } diff --git a/src/Raven.Server/Documents/ETL/Providers/Queue/Handlers/Processors/QueueEtlHandlerProcessorForTestAzureQueueStorageConnection.cs b/src/Raven.Server/Documents/ETL/Providers/Queue/Handlers/Processors/QueueEtlHandlerProcessorForTestAzureQueueStorageConnection.cs index 5cabcd10a873..e8fd39dd6980 100644 --- a/src/Raven.Server/Documents/ETL/Providers/Queue/Handlers/Processors/QueueEtlHandlerProcessorForTestAzureQueueStorageConnection.cs +++ b/src/Raven.Server/Documents/ETL/Providers/Queue/Handlers/Processors/QueueEtlHandlerProcessorForTestAzureQueueStorageConnection.cs @@ -28,9 +28,8 @@ public override async ValueTask ExecuteAsync() try { string authenticationJson = await new StreamReader(HttpContext.Request.Body).ReadToEndAsync(); - Authentication authentication = JsonConvert.DeserializeObject(authenticationJson); - - var connectionSettings = new AzureQueueStorageConnectionSettings() { Authentication = authentication }; + AzureQueueStorageConnectionSettings connectionSettings = + JsonConvert.DeserializeObject(authenticationJson); QueueServiceClient client = QueueBrokerConnectionHelper.CreateAzureQueueStorageServiceClient(connectionSettings); diff --git a/src/Raven.Server/Documents/ETL/Providers/Queue/QueueBrokerConnectionHelper.cs b/src/Raven.Server/Documents/ETL/Providers/Queue/QueueBrokerConnectionHelper.cs index 55a1e1eef95c..9c947e006cf0 100644 --- a/src/Raven.Server/Documents/ETL/Providers/Queue/QueueBrokerConnectionHelper.cs +++ b/src/Raven.Server/Documents/ETL/Providers/Queue/QueueBrokerConnectionHelper.cs @@ -94,24 +94,24 @@ public static QueueClient CreateAzureQueueStorageClient( { QueueClient queueClient; - if (azureQueueStorageConnectionSettings.Authentication.ConnectionString != null) + if (azureQueueStorageConnectionSettings.ConnectionString != null) { - queueClient = new QueueClient(azureQueueStorageConnectionSettings.Authentication.ConnectionString, + queueClient = new QueueClient(azureQueueStorageConnectionSettings.ConnectionString, queueName); } - else if (azureQueueStorageConnectionSettings.Authentication.EntraId != null) + else if (azureQueueStorageConnectionSettings.EntraId != null) { var queueUri = new Uri($"{azureQueueStorageConnectionSettings.GetStorageUrl()}{queueName}"); queueClient = new QueueClient( queueUri, new ClientSecretCredential( - azureQueueStorageConnectionSettings.Authentication.EntraId.TenantId, - azureQueueStorageConnectionSettings.Authentication.EntraId.ClientId, - azureQueueStorageConnectionSettings.Authentication.EntraId.ClientSecret)); + azureQueueStorageConnectionSettings.EntraId.TenantId, + azureQueueStorageConnectionSettings.EntraId.ClientId, + azureQueueStorageConnectionSettings.EntraId.ClientSecret)); } - else if(azureQueueStorageConnectionSettings.Authentication.Passwordless) + else if(azureQueueStorageConnectionSettings.Passwordless) { var queueUri = new Uri($"{azureQueueStorageConnectionSettings.GetStorageUrl()}{queueName}"); queueClient = new QueueClient(queueUri, new DefaultAzureCredential()); @@ -129,22 +129,22 @@ public static QueueServiceClient CreateAzureQueueStorageServiceClient( { QueueServiceClient queueServiceClient = null; - if (azureQueueStorageConnectionSettings.Authentication.ConnectionString != null) + if (azureQueueStorageConnectionSettings.ConnectionString != null) { queueServiceClient = - new QueueServiceClient(azureQueueStorageConnectionSettings.Authentication.ConnectionString); + new QueueServiceClient(azureQueueStorageConnectionSettings.ConnectionString); } - else if (azureQueueStorageConnectionSettings.Authentication.EntraId != null) + else if (azureQueueStorageConnectionSettings.EntraId != null) { var queueUri = new Uri(azureQueueStorageConnectionSettings.GetStorageUrl()); queueServiceClient = new QueueServiceClient( queueUri, new ClientSecretCredential( - azureQueueStorageConnectionSettings.Authentication.EntraId.TenantId, - azureQueueStorageConnectionSettings.Authentication.EntraId.ClientId, - azureQueueStorageConnectionSettings.Authentication.EntraId.ClientSecret)); + azureQueueStorageConnectionSettings.EntraId.TenantId, + azureQueueStorageConnectionSettings.EntraId.ClientId, + azureQueueStorageConnectionSettings.EntraId.ClientSecret)); } return queueServiceClient; diff --git a/src/Raven.Server/Raven.Server.csproj b/src/Raven.Server/Raven.Server.csproj index a4557c9c403a..d199ed7aee2e 100644 --- a/src/Raven.Server/Raven.Server.csproj +++ b/src/Raven.Server/Raven.Server.csproj @@ -154,10 +154,7 @@ - - - - + diff --git a/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTestBase.cs b/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTestBase.cs index 2fcca67805c0..da4d125918e1 100644 --- a/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTestBase.cs +++ b/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTestBase.cs @@ -76,10 +76,7 @@ protected QueueEtlConfiguration SetupQueueEtlToAzureQueueStorageOnline(DocumentS BrokerType = QueueBrokerType.AzureQueueStorage, AzureQueueStorageConnectionSettings = new AzureQueueStorageConnectionSettings { - Authentication = new global::Raven.Client.Documents.Operations.ETL.Queue.Authentication - { - ConnectionString = ConnectionString - } + ConnectionString = ConnectionString } }); return config; diff --git a/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTests.cs b/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTests.cs index 688c902287d6..b3696e86caf0 100644 --- a/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTests.cs +++ b/test/SlowTests/Server/Documents/ETL/Queue/AzureQueueStorageEtlTests.cs @@ -209,14 +209,7 @@ public void Error_if_script_is_empty() Name = "Foo", BrokerType = QueueBrokerType.AzureQueueStorage, AzureQueueStorageConnectionSettings = - new AzureQueueStorageConnectionSettings() - { - Authentication = - new global::Raven.Client.Documents.Operations.ETL.Queue.Authentication() - { - ConnectionString = ConnectionString - } - } + new AzureQueueStorageConnectionSettings { ConnectionString = ConnectionString } }); List errors; @@ -251,14 +244,7 @@ await session.StoreAsync(new Order Name = "simulate", BrokerType = QueueBrokerType.AzureQueueStorage, AzureQueueStorageConnectionSettings = - new AzureQueueStorageConnectionSettings() - { - Authentication = - new global::Raven.Client.Documents.Operations.ETL.Queue.Authentication() - { - ConnectionString = ConnectionString - } - } + new AzureQueueStorageConnectionSettings { ConnectionString = ConnectionString } })); Assert.NotNull(result1.RaftCommandIndex); @@ -374,7 +360,7 @@ public async Task ShouldImportTask() Assert.True(destinationRecord.QueueEtls[0].Queues[0].DeleteProcessedDocuments); } } - + [RavenFact(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding | RavenTestCategory.Etl)] public async Task ShouldSkipUnsupportedFeaturesInShardingOnImport_RabbitMqEtl() { @@ -382,20 +368,15 @@ public async Task ShouldSkipUnsupportedFeaturesInShardingOnImport_RabbitMqEtl() using (var dstStore = Sharding.GetDocumentStore()) { var config = SetupQueueEtlToAzureQueueStorageOnline(srcStore, - DefaultScript, DefaultCollections, new List() - { - new() - { - Name = "Orders", - DeleteProcessedDocuments = true - } - }, connectionString: ConnectionString); + DefaultScript, DefaultCollections, + new List() { new() { Name = "Orders", DeleteProcessedDocuments = true } }, + connectionString: ConnectionString); var record = await srcStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(srcStore.Database)); Assert.NotNull(record.QueueEtls); Assert.Equal(1, record.QueueEtls.Count); - + var exportFile = GetTempFileName(); var exportOperation = await srcStore.Smuggler.ExportAsync(new DatabaseSmugglerExportOptions(), exportFile);