From 239a7b550e91a09c896d56ea6f473b23e9e02245 Mon Sep 17 00:00:00 2001 From: "Pohsiang (John) Hsu" Date: Wed, 21 Feb 2024 16:18:36 -0800 Subject: [PATCH] cache BlobServiceClient to mitigate Azure Storage Management API throttling --- migrationTool/ams/AssetAnalyzer.cs | 8 ++--- migrationTool/ams/AssetMigrator.cs | 4 +-- migrationTool/ams/AzureResourceProvider.cs | 41 ++++++++++++++-------- migrationTool/ams/CleanupCommand.cs | 5 ++- migrationTool/ams/ResetCommand.cs | 7 ++-- migrationTool/ams/StorageMigrator.cs | 3 +- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/migrationTool/ams/AssetAnalyzer.cs b/migrationTool/ams/AssetAnalyzer.cs index c5bc292..2a2d308 100644 --- a/migrationTool/ams/AssetAnalyzer.cs +++ b/migrationTool/ams/AssetAnalyzer.cs @@ -144,7 +144,8 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) var assetTypes = new ConcurrentDictionary(); if (!isAMSAcc) { - var (storageClient, accountId) = await _resourceProvider.GetStorageAccount(_analysisOptions.AccountName, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(_analysisOptions.AccountName, cancellationToken); + var (storageClient, accountId) = _resourceProvider.GetBlobServiceClient(_analysisOptions.AccountName); if (storageClient == null) { _logger.LogError("No valid storage account was found."); @@ -167,7 +168,6 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) var writer = channel.Writer; await MigrateInParallel(containers, filteredList, async (container, cancellationToken) => { - // var storage = await _resourceProvider.GetStorageAccountAsync(account, asset, cancellationToken); var result = await AnalyzeAsync(container, storageClient, cancellationToken); var assetType = result.AssetType ?? "unknown"; assetTypes.AddOrUpdate(assetType, 1, (key, value) => Interlocked.Increment(ref value)); @@ -197,7 +197,7 @@ await MigrateInParallel(containers, filteredList, async (container, cancellation double totalAssets = await QueryMetricAsync(account.Id.ToString(), "AssetCount", cancellationToken); _logger.LogInformation("The total asset count of the media account is {count}.", totalAssets); - await _resourceProvider.SetStorageResourceGroupsAsync(account, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(account, cancellationToken); var assets = account.GetMediaAssets() .GetAllAsync(resourceFilter, cancellationToken: cancellationToken); statistics = new AssetStats(); @@ -221,7 +221,7 @@ await MigrateInParallel(containers, filteredList, async (container, cancellation var writer = channel.Writer; await MigrateInParallel(assets, filteredList, async (asset, cancellationToken) => { - var storage = await _resourceProvider.GetStorageAccountAsync(account, asset, cancellationToken); + var storage = _resourceProvider.GetBlobServiceClient(asset); var result = await AnalyzeAsync(asset, storage, cancellationToken); var assetType = result.AssetType ?? "unknown"; assetTypes.AddOrUpdate(assetType, 1, (key, value) => Interlocked.Increment(ref value)); diff --git a/migrationTool/ams/AssetMigrator.cs b/migrationTool/ams/AssetMigrator.cs index 32eab08..5c9c92c 100644 --- a/migrationTool/ams/AssetMigrator.cs +++ b/migrationTool/ams/AssetMigrator.cs @@ -55,7 +55,7 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) var orderBy = "properties/created"; - await _resourceProvider.SetStorageResourceGroupsAsync(account, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(account, cancellationToken); var assets = account.GetMediaAssets().GetAllAsync(resourceFilter, orderby: orderBy, cancellationToken: cancellationToken); List? filteredList = null; @@ -85,7 +85,7 @@ private async Task MigrateAsync(MediaServicesAccountResource account var stats = new AssetStats(); await MigrateInParallel(assets, filteredList, async (asset, cancellationToken) => { - var storage = await _resourceProvider.GetStorageAccountAsync(account, asset, cancellationToken); + var storage = _resourceProvider.GetBlobServiceClient(asset); var result = await MigrateAsync(account, storage, asset, cancellationToken); stats.Update(result); diff --git a/migrationTool/ams/AzureResourceProvider.cs b/migrationTool/ams/AzureResourceProvider.cs index e05be25..bcec55b 100644 --- a/migrationTool/ams/AzureResourceProvider.cs +++ b/migrationTool/ams/AzureResourceProvider.cs @@ -16,7 +16,7 @@ class AzureResourceProvider protected readonly TokenCredential _credentials; protected readonly ArmClient _armClient; - private Dictionary _storageResourceGroups; + private Dictionary _storageAccountResources; public AzureResourceProvider(TokenCredential credential, GlobalOptions options) { @@ -29,10 +29,10 @@ public AzureResourceProvider(TokenCredential credential, GlobalOptions options) options.SubscriptionId, options.ResourceGroup); _resourceGroup = _armClient.GetResourceGroupResource(resourceGroupId); - _storageResourceGroups = new Dictionary(); + _storageAccountResources = new Dictionary(); } - public async Task SetStorageResourceGroupsAsync(MediaServicesAccountResource account, CancellationToken cancellationToken) + public async Task SetStorageAccountResourcesAsync(MediaServicesAccountResource account, CancellationToken cancellationToken) { IList storageAccounts; var mediaServiceResource = await account.GetAsync(cancellationToken: cancellationToken); @@ -47,11 +47,20 @@ public async Task SetStorageResourceGroupsAsync(MediaServicesAccountResource acc storageAccountId.SubscriptionId!, storageAccountId.ResourceGroupName!); var resourceGroup = _armClient.GetResourceGroupResource(resourceGroupId); - _storageResourceGroups.Add(storageAccountId.Name, resourceGroup); + StorageAccountResource storage = await resourceGroup.GetStorageAccountAsync(storageAccountId.Name, + cancellationToken: cancellationToken); + _storageAccountResources.Add(storageAccountId.Name, storage); } } } + public async Task SetStorageAccountResourcesAsync(string storageAccountName, CancellationToken cancellationToken) + { + StorageAccountResource storage = await _resourceGroup.GetStorageAccountAsync(storageAccountName, + cancellationToken: cancellationToken); + _storageAccountResources.Add(storageAccountName, storage); + } + public async Task GetMediaAccountAsync( string mediaAccountName, CancellationToken cancellationToken) @@ -60,22 +69,24 @@ public async Task GetMediaAccountAsync( mediaAccountName, cancellationToken); } - public async Task GetStorageAccountAsync( - MediaServicesAccountResource account, - MediaAssetResource asset, - CancellationToken cancellationToken) + public BlobServiceClient GetBlobServiceClient(MediaAssetResource asset) { string assetStorageAccountName = asset.Data.StorageAccountName; - _storageResourceGroups.TryGetValue(asset.Data.StorageAccountName, out var rg); - var resource = await rg.GetStorageAccountAsync(asset.Data.StorageAccountName, cancellationToken: cancellationToken); - return GetStorageAccount(resource); + if (_storageAccountResources.TryGetValue(assetStorageAccountName, out var resource)) + { + return GetStorageAccount(resource); + } + throw new Exception($"Failed to get BlobServiceClient for storage account {assetStorageAccountName}."); } - public async Task<(BlobServiceClient, ResourceIdentifier)> GetStorageAccount(string storageAccountName, CancellationToken cancellationToken) + public (BlobServiceClient, ResourceIdentifier) GetBlobServiceClient(string storageAccountName) { - StorageAccountResource storage = - await _resourceGroup.GetStorageAccountAsync(storageAccountName, cancellationToken: cancellationToken); - return (GetStorageAccount(storage), storage.Id); + string assetStorageAccountName = storageAccountName; + if (_storageAccountResources.TryGetValue(assetStorageAccountName, out var resource)) + { + return (GetStorageAccount(resource), resource.Id); + } + throw new Exception($"Failed to get BlobServiceClient for storage account {assetStorageAccountName}."); } private BlobServiceClient GetStorageAccount(StorageAccountResource storage) diff --git a/migrationTool/ams/CleanupCommand.cs b/migrationTool/ams/CleanupCommand.cs index 95ba1e0..9274cbf 100644 --- a/migrationTool/ams/CleanupCommand.cs +++ b/migrationTool/ams/CleanupCommand.cs @@ -56,7 +56,7 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) var resourceFilter = _options.IsCleanUpAccount ? null : GetAssetResourceFilter(_options.ResourceFilter, null, null); var orderBy = "properties/created"; - await _resourceProvider.SetStorageResourceGroupsAsync(account, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(account, cancellationToken); assets = account.GetMediaAssets() .GetAllAsync(resourceFilter, orderby: orderBy, cancellationToken: cancellationToken); List? assetList = await assets.ToListAsync(cancellationToken); @@ -160,8 +160,7 @@ private async Task CleanUpAssetAsync(bool isForcedelete, MediaServicesAcco { try { - - var storage = await _resourceProvider.GetStorageAccountAsync(account, asset, cancellationToken); + var storage = _resourceProvider.GetBlobServiceClient(asset); var container = storage.GetContainer(asset); if (!await container.ExistsAsync(cancellationToken)) { diff --git a/migrationTool/ams/ResetCommand.cs b/migrationTool/ams/ResetCommand.cs index bbba3c3..5228196 100644 --- a/migrationTool/ams/ResetCommand.cs +++ b/migrationTool/ams/ResetCommand.cs @@ -35,7 +35,8 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) var (isAMSAcc, account) = await IsAMSAccountAsync(_options.AccountName, cancellationToken); if (!isAMSAcc) { - var (storageClient, accountId) = await _resourceProvider.GetStorageAccount(_options.AccountName, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(_options.AccountName, cancellationToken); + var (storageClient, accountId) = _resourceProvider.GetBlobServiceClient(_options.AccountName); if (storageClient == null) { _logger.LogError("No valid storage account was found."); @@ -64,14 +65,14 @@ public override async Task MigrateAsync(CancellationToken cancellationToken) throw new Exception("No valid media account was found."); } _logger.LogInformation("Begin reset assets on account: {name}", account.Data.Name); - await _resourceProvider.SetStorageResourceGroupsAsync(account, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(account, cancellationToken); AsyncPageable assets = account.GetMediaAssets() .GetAllAsync(cancellationToken: cancellationToken); List? assetList = await assets.ToListAsync(cancellationToken); int resetAssetCount = 0; foreach (var asset in assetList) { - var (storage, _) = await _resourceProvider.GetStorageAccount(asset.Data.StorageAccountName, cancellationToken); + var storage = _resourceProvider.GetBlobServiceClient(asset); var container = storage.GetContainer(asset); if (!await container.ExistsAsync(cancellationToken)) { diff --git a/migrationTool/ams/StorageMigrator.cs b/migrationTool/ams/StorageMigrator.cs index df26c6a..a952a81 100644 --- a/migrationTool/ams/StorageMigrator.cs +++ b/migrationTool/ams/StorageMigrator.cs @@ -35,7 +35,8 @@ public StorageMigrator( public override async Task MigrateAsync(CancellationToken cancellationToken) { var watch = Stopwatch.StartNew(); - var (storageClient, accountId) = await _resourceProvider.GetStorageAccount(_storageOptions.AccountName, cancellationToken); + await _resourceProvider.SetStorageAccountResourcesAsync(_storageOptions.AccountName, cancellationToken); + var (storageClient, accountId) = _resourceProvider.GetBlobServiceClient(_storageOptions.AccountName); _logger.LogInformation("Begin migration of containers from account: {name}", storageClient.AccountName); double totalContainers = await GetStorageBlobMetricAsync(accountId, cancellationToken); _logger.LogInformation("The total count of containers of the storage account is {count}.", totalContainers);