Index ExtendedqueryTag on dicom update for a study (#3210)

* Index ExtendedqueryTag on dicom update for a study * Fix ext sproc * Add study level filter * Fix comments * Fix comment
microsoft · Nov 22, 2023 · 028f8fa · 028f8fa
1 parent 981a2c6
commit 028f8fa
Show file tree

Hide file tree

Showing 28 changed files with 6,560 additions and 169 deletions.
diff --git a/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs b/src/Microsoft.Health.Dicom.Core/Features/Store/IIndexDataStore.cs
@@ -163,9 +163,10 @@ public interface IIndexDataStore
     /// <param name="studyInstanceUid"></param>
     /// <param name="dicomDataset">The DICOM dataset to index.</param>
     /// <param name="instanceMetadataList">A list of instance metadata to use to update file properties for newly stored blob file from "update"</param>
+    /// <param name="queryTags">Queryable dicom tags</param>
     /// <param name="cancellationToken">The cancellation token.</param>
     /// <returns>A task that represents the asynchronous add operation.</returns>
-    Task EndUpdateInstanceAsync(int partitionKey, string studyInstanceUid, DicomDataset dicomDataset, IReadOnlyList<InstanceMetadata> instanceMetadataList, CancellationToken cancellationToken = default);
+    Task EndUpdateInstanceAsync(int partitionKey, string studyInstanceUid, DicomDataset dicomDataset, IReadOnlyList<InstanceMetadata> instanceMetadataList, IEnumerable<QueryTag> queryTags, CancellationToken cancellationToken = default);
 
     /// <summary>
     /// Asynchronously updates DICOM instance HasFrameMetadata to 1

diff --git a/src/Microsoft.Health.Dicom.Functions.App/host.json b/src/Microsoft.Health.Dicom.Functions.App/host.json
@@ -89,7 +89,7 @@
     "Frequency": "0 0 * * *",
     "MinimumAgeDays": 7,
     "Statuses": [ "Completed" ],
-    "ExcludeFunctions": [ "MigratingFrameRangeFilesAsync", "UpdateInstancesV3Async", "UpdateInstancesV4Async" ]
+    "ExcludeFunctions": [ "MigratingFrameRangeFilesAsync", "UpdateInstancesV4Async", "UpdateInstancesV5Async" ]
   },
   "SqlServer": {
     "Retry": {

diff --git a/src/Microsoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.Activity.cs b/src/Microsoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.Activity.cs
@@ -3,6 +3,7 @@
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
@@ -11,6 +12,7 @@
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Health.Dicom.Core.Exceptions;
 using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.ExtendedQueryTag;
 using Microsoft.Health.Dicom.Core.Features.Model;
 using Microsoft.Health.Dicom.Core.Features.Partitioning;
 using Microsoft.Health.Dicom.Functions.Update.Models;
@@ -137,14 +139,14 @@ public async Task GivenCompleteInstanceArgument_WhenCompleting_ThenShouldComplet
         var studyInstanceUid = TestUidGenerator.Generate();
 
         var instanceMetadataList = new List<InstanceMetadata>();
-        _indexStore.EndUpdateInstanceAsync(Partition.DefaultKey, studyInstanceUid, new DicomDataset(), instanceMetadataList, CancellationToken.None).Returns(Task.CompletedTask);
+        _indexStore.EndUpdateInstanceAsync(Partition.DefaultKey, studyInstanceUid, new DicomDataset(), instanceMetadataList, Array.Empty<QueryTag>(), CancellationToken.None).Returns(Task.CompletedTask);
 
         var ds = new DicomDataset
         {
             { DicomTag.PatientName, "Patient Name" }
         };
 
-        await _updateDurableFunction.CompleteUpdateStudyV3Async(
+        await _updateDurableFunction.CompleteUpdateStudyV4Async(
             new CompleteStudyArgumentsV2(
                 Partition.DefaultKey,
                 studyInstanceUid,
@@ -154,7 +156,7 @@ await _updateDurableFunction.CompleteUpdateStudyV3Async(
 
         await _indexStore
             .Received(1)
-            .EndUpdateInstanceAsync(Partition.DefaultKey, studyInstanceUid, Arg.Is<DicomDataset>(x => x.GetSingleValue<string>(DicomTag.PatientName) == "Patient Name"), instanceMetadataList, CancellationToken.None);
+            .EndUpdateInstanceAsync(Partition.DefaultKey, studyInstanceUid, Arg.Is<DicomDataset>(x => x.GetSingleValue<string>(DicomTag.PatientName) == "Patient Name"), instanceMetadataList, Arg.Any<IEnumerable<QueryTag>>(), CancellationToken.None);
     }
 
     [Fact]

diff --git a/...osoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.Orchestration.cs b/...osoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.Orchestration.cs
@@ -87,7 +87,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstances_ThenComple
             .Returns(new UpdateInstanceResponse(instanceMetadataList, new List<string>()));
         context
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>())
             .Returns(Task.CompletedTask);
@@ -105,7 +105,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstances_ThenComple
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         context
@@ -127,7 +127,7 @@ await context
         await context
             .Received(1)
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>());
         await context
@@ -208,7 +208,7 @@ public async Task GivenV4OrchestrationWithInputAndExternalStoreEnabled_WhenUpdat
             .Returns(new UpdateInstanceResponse(instanceMetadataList, new List<string>()));
         context
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Is(GetPredicate(expectedInput.Partition.Key, studyInstanceUid, expectedInput.ChangeDataset, instanceMetadataList)))
             .Returns(Task.CompletedTask);
@@ -226,7 +226,7 @@ public async Task GivenV4OrchestrationWithInputAndExternalStoreEnabled_WhenUpdat
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunctionWithExternalStore.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunctionWithExternalStore.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         context
@@ -248,7 +248,7 @@ await context
         await context
             .Received(1)
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Is(GetPredicate(expectedInput.Partition.Key, studyInstanceUid, expectedInput.ChangeDataset, instanceMetadataList)));
         context
@@ -329,7 +329,7 @@ public async Task GivenV4OrchestrationWithInputAndExternalStoreNotEnabled_WhenUp
             .Returns(new UpdateInstanceResponse(instanceMetadataList, new List<string>()));
         context
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Is(GetPredicate(expectedInput.Partition.Key, studyInstanceUid, expectedInput.ChangeDataset, new List<InstanceMetadata>())))
             .Returns(Task.CompletedTask);
@@ -347,7 +347,7 @@ public async Task GivenV4OrchestrationWithInputAndExternalStoreNotEnabled_WhenUp
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         context
@@ -369,7 +369,7 @@ await context
         await context
             .Received(1)
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Is(GetPredicate(expectedInput.Partition.Key, studyInstanceUid, expectedInput.ChangeDataset, null, expectEmptyList: true)));
         context
@@ -426,7 +426,7 @@ public async Task GivenV4OrchestrationWithNoInstancesFound_WhenUpdatingInstances
             .Returns(new UpdateInstanceResponse(instanceMetadataList, new List<string>()));
         context
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>())
             .Returns(Task.CompletedTask);
@@ -444,7 +444,7 @@ public async Task GivenV4OrchestrationWithNoInstancesFound_WhenUpdatingInstances
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         context
@@ -465,7 +465,7 @@ await context
         await context
             .DidNotReceive()
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>());
         context
@@ -520,7 +520,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstancesWithExcepti
             .Returns(expectedInput);
 
         // Invoke the orchestration
-        await Assert.ThrowsAsync<OperationErrorException>(() => _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance));
+        await Assert.ThrowsAsync<OperationErrorException>(() => _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance));
 
         // Assert behavior
         context
@@ -541,7 +541,7 @@ await context
         await context
             .DidNotReceive()
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>());
         context
@@ -629,7 +629,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstancesWithExcepti
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         await context
@@ -718,7 +718,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstancesWithDataSto
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         await context
@@ -792,7 +792,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstances_ThenComple
             .Returns(new UpdateInstanceResponse(instanceMetadataList, new List<string>()));
         context
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>())
             .Returns(Task.CompletedTask);
@@ -804,7 +804,7 @@ public async Task GivenV4OrchestrationWithInput_WhenUpdatingInstances_ThenComple
             .Returns(Task.CompletedTask);
 
         // Invoke the orchestration
-        await _updateDurableFunction.UpdateInstancesV4Async(context, NullLogger.Instance);
+        await _updateDurableFunction.UpdateInstancesV5Async(context, NullLogger.Instance);
 
         // Assert behavior
         context
@@ -825,7 +825,7 @@ await context
         await context
             .Received(1)
             .CallActivityWithRetryAsync(
-                nameof(UpdateDurableFunction.CompleteUpdateStudyV3Async),
+                nameof(UpdateDurableFunction.CompleteUpdateStudyV4Async),
                 _options.RetryOptions,
                 Arg.Any<CompleteStudyArgumentsV2>());
         context

diff --git a/src/Microsoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.cs b/src/Microsoft.Health.Dicom.Functions.UnitTests/Update/UpdateDurableFunctionTests.cs
@@ -9,6 +9,7 @@
 using Microsoft.Extensions.Options;
 using Microsoft.Health.Dicom.Core.Configs;
 using Microsoft.Health.Dicom.Core.Features.Common;
+using Microsoft.Health.Dicom.Core.Features.ExtendedQueryTag;
 using Microsoft.Health.Dicom.Core.Features.Retrieve;
 using Microsoft.Health.Dicom.Core.Features.Store;
 using Microsoft.Health.Dicom.Core.Features.Telemetry;
@@ -56,6 +57,7 @@ public UpdateDurableFunctionTests()
             _metadataStore,
             _fileStore,
             _updateInstanceService,
+            Substitute.For<IQueryTagService>(),
             _updateMeter,
             Options.Create(_jsonSerializerOptions),
             Options.Create(new FeatureConfiguration()));
@@ -66,6 +68,7 @@ public UpdateDurableFunctionTests()
             _metadataStore,
             _fileStore,
             _updateInstanceService,
+            Substitute.For<IQueryTagService>(),
             _updateMeter,
             Options.Create(_jsonSerializerOptions),
             Options.Create(new FeatureConfiguration { EnableExternalStore = true, }));

diff --git a/src/Microsoft.Health.Dicom.Functions/Update/Models/UpdateInstanceBlobArgumentsV2.cs b/src/Microsoft.Health.Dicom.Functions/Update/Models/UpdateInstanceBlobArgumentsV2.cs
@@ -1,4 +1,4 @@
-// -------------------------------------------------------------------------------------------------
+// -------------------------------------------------------------------------------------------------
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
 // -------------------------------------------------------------------------------------------------
@@ -11,7 +11,7 @@
 namespace Microsoft.Health.Dicom.Functions.Update.Models;
 
 /// <summary>
-/// Represents input to <see cref="UpdateDurableFunction.UpdateInstanceBlobsV2Async"/>
+/// Represents input to <see cref="UpdateDurableFunction.UpdateInstanceBlobsV3Async"/>
 /// </summary>
 public class UpdateInstanceBlobArgumentsV2
 {