From 8583c70b68e69ada79fb49b0767adb5dbdfb69c4 Mon Sep 17 00:00:00 2001 From: John Stairs Date: Sun, 12 May 2019 06:35:46 -0700 Subject: [PATCH] Compartment indexing and TVP generator codegen (#457) --- .../AssemblyInfo.cs | 1 + ...rBuilderSqlServerRegistrationExtensions.cs | 23 ++ .../AssemblyInfo.cs | 2 + .../Features/Health/SqlServerHealthCheck.cs | 5 +- .../Features/Schema/Migrations/1.sql | 80 +++++- ...ProcedureTableValuedParametersGenerator.cs | 17 ++ .../ITableValuedParameterRowGenerator.cs | 20 ++ .../Features/Schema/Model/V1.Generated.cs | 241 +++++++++++++----- .../Storage/SqlServerFhirDataStore.cs | 30 +-- .../Features/Storage/SqlServerFhirModel.cs | 36 ++- .../CompartmentAssignmentRowGenerator.cs | 98 +++++++ .../ResourceWriteClaimRowGenerator.cs | 30 +++ .../SqlServerFhirStorageTestsFixture.cs | 12 +- .../Sql/CreateProcedureVisitor.cs | 229 ++++++++++++++++- .../Sql/CreateTableTypeVisitor.cs | 7 +- .../Sql/CreateTableVisitor.cs | 9 +- .../Sql/MemberSorting.cs | 51 ++++ .../Sql/SqlModelGenerator.cs | 57 +---- .../Sql/SqlVisitor.cs | 18 +- 19 files changed, 797 insertions(+), 169 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/IStoredProcedureTableValuedParametersGenerator.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/ITableValuedParameterRowGenerator.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/CompartmentAssignmentRowGenerator.cs create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/ResourceWriteClaimRowGenerator.cs create mode 100644 tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/MemberSorting.cs diff --git a/src/Microsoft.Health.Fhir.SqlServer.Api/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.SqlServer.Api/AssemblyInfo.cs index e000878c29..fde0b981ef 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.Api/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.Api/AssemblyInfo.cs @@ -7,4 +7,5 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.Api.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Tests.Integration")] [assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Fhir.SqlServer.Api/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer.Api/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index 19c4bf384f..af1e41fcef 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.Api/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.Api/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Linq; using EnsureThat; using Microsoft.Extensions.Configuration; using Microsoft.Health.Extensions.DependencyInjection; @@ -11,6 +12,7 @@ using Microsoft.Health.Fhir.SqlServer.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Health; using Microsoft.Health.Fhir.SqlServer.Features.Schema; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Extensions.DependencyInjection @@ -71,7 +73,28 @@ public static IServiceCollection AddExperimentalSqlServer(this IServiceCollectio // During normal usage, the controller should be automatically discovered. serviceCollection.AddMvc().AddApplicationPart(typeof(SchemaController).Assembly); + AddSqlServerTableRowParameterGenerators(serviceCollection); + return serviceCollection; } + + internal static void AddSqlServerTableRowParameterGenerators(this IServiceCollection serviceCollection) + { + foreach (var type in typeof(SqlServerFhirDataStore).Assembly.GetTypes().Where(t => t.IsClass && !t.IsAbstract)) + { + foreach (var interfaceType in type.GetInterfaces()) + { + if (interfaceType.IsGenericType && interfaceType.GetGenericTypeDefinition() == typeof(IStoredProcedureTableValuedParametersGenerator<,>)) + { + serviceCollection.AddSingleton(type); + } + + if (interfaceType.IsGenericType && interfaceType.GetGenericTypeDefinition() == typeof(ITableValuedParameterRowGenerator<,>)) + { + serviceCollection.AddSingleton(interfaceType, type); + } + } + } + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs index 7d063fafbf..5ffb815043 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/AssemblyInfo.cs @@ -6,5 +6,7 @@ using System.Resources; using System.Runtime.CompilerServices; +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.Api")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Tests.Integration")] [assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Health/SqlServerHealthCheck.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Health/SqlServerHealthCheck.cs index 2ceebed995..23790fa287 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Health/SqlServerHealthCheck.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Health/SqlServerHealthCheck.cs @@ -11,7 +11,6 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.SqlServer.Configs; -using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Health { @@ -21,9 +20,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Health public class SqlServerHealthCheck : IHealthCheck { private readonly SqlServerDataStoreConfiguration _configuration; - private readonly ILogger _logger; + private readonly ILogger _logger; - public SqlServerHealthCheck(SqlServerDataStoreConfiguration configuration, ILogger logger) + public SqlServerHealthCheck(SqlServerDataStoreConfiguration configuration, ILogger logger) { EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(logger, nameof(logger)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/1.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/1.sql index e23e507d89..be573f10ed 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/1.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/1.sql @@ -208,7 +208,7 @@ CREATE UNIQUE CLUSTERED INDEX IXC_Claim on dbo.ClaimType Name ) -CREATE TYPE dbo.ResourceWriteClaimTableType AS TABLE +CREATE TYPE dbo.ResourceWriteClaimTableType_1 AS TABLE ( ClaimTypeId tinyint NOT NULL, ClaimValue nvarchar(128) NOT NULL @@ -227,6 +227,52 @@ CREATE CLUSTERED INDEX IXC_LastModifiedClaim on dbo.ResourceWriteClaim ClaimTypeId ) +/************************************************************* + Compartments +**************************************************************/ + +CREATE TABLE dbo.CompartmentType +( + CompartmentTypeId tinyint IDENTITY(1,1) NOT NULL, + Name varchar(128) NOT NULL +) + +CREATE UNIQUE CLUSTERED INDEX IXC_CompartmentType on dbo.CompartmentType +( + Name +) + +CREATE TYPE dbo.CompartmentAssignmentTableType_1 AS TABLE +( + CompartmentTypeId tinyint NOT NULL, + ReferenceResourceId varchar(64) NOT NULL +) + +CREATE TABLE dbo.CompartmentAssignment +( + ResourceSurrogateId bigint NOT NULL, + CompartmentTypeId tinyint NOT NULL, + ReferenceResourceId varchar(64) NOT NULL, + IsHistory bit NOT NULL, +) WITH (DATA_COMPRESSION = PAGE) + +CREATE CLUSTERED INDEX IXC_CompartmentAssignment +ON dbo.CompartmentAssignment +( + ResourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId +) + +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId +ON dbo.CompartmentAssignment +( + CompartmentTypeId, + ReferenceResourceId +) +WHERE IsHistory = 0 +WITH (DATA_COMPRESSION = PAGE) + GO @@ -286,7 +332,8 @@ CREATE PROCEDURE dbo.UpsertResource @keepHistory bit, @requestMethod varchar(10), @rawResource varbinary(max), - @resourceWriteClaims dbo.ResourceWriteClaimTableType READONLY + @resourceWriteClaims dbo.ResourceWriteClaimTableType_1 READONLY, + @compartmentAssignments dbo.CompartmentAssignmentTableType_1 READONLY AS SET NOCOUNT ON @@ -332,7 +379,30 @@ AS END ELSE BEGIN -- There is a previous version - SET @version = (select (Version + 1) from @previousVersion) + DECLARE @previousResourceSurrogateId bigint + + SELECT @version = (Version + 1), @previousResourceSurrogateId = ResourceSurrogateId + FROM @previousVersion + + IF (@keepHistory = 1) BEGIN + + -- note there is no IsHistory column on ResourceWriteClaim since we do not query it + + UPDATE dbo.CompartmentAssignment + SET IsHistory = 1 + WHERE ResourceSurrogateId = @previousResourceSurrogateId + + END + ELSE BEGIN + + DELETE FROM dbo.ResourceWriteClaim + WHERE ResourceSurrogateId = @previousResourceSurrogateId + + DELETE FROM dbo.CompartmentAssignment + WHERE ResourceSurrogateId = @previousResourceSurrogateId + + + END END @@ -351,6 +421,10 @@ AS (ResourceSurrogateId, ClaimTypeId, ClaimValue) SELECT @resourceSurrogateId, ClaimTypeId, ClaimValue from @resourceWriteClaims + INSERT INTO dbo.CompartmentAssignment + (ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) + SELECT @resourceSurrogateId, CompartmentTypeId, ReferenceResourceId, 0 + FROM @compartmentAssignments select @version diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/IStoredProcedureTableValuedParametersGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/IStoredProcedureTableValuedParametersGenerator.cs new file mode 100644 index 0000000000..9b3fbe4760 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/IStoredProcedureTableValuedParametersGenerator.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.SqlServer.Features.Schema.Model +{ + /// + /// Generates the full set of table-valued parameters for a stored procedure. + /// + /// The type of the input + /// The type of the output. Intended to be a struct with properties for each TVP + internal interface IStoredProcedureTableValuedParametersGenerator + { + TOutput Generate(TInput input); + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/ITableValuedParameterRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/ITableValuedParameterRowGenerator.cs new file mode 100644 index 0000000000..c0cff6e0d5 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/ITableValuedParameterRowGenerator.cs @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Schema.Model +{ + /// + /// Generates a sequence of row structs for a table-valued parameter. + /// + /// The input type + /// The row struct type + internal interface ITableValuedParameterRowGenerator + where TRow : struct + { + IEnumerable GenerateRows(TInput input); + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/V1.Generated.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/V1.Generated.cs index 11d1bb16fe..f4a2bc6127 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/V1.Generated.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Model/V1.Generated.cs @@ -10,12 +10,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema.Model { internal class V1 { - internal readonly static HardDeleteResourceProcedure HardDeleteResource = new HardDeleteResourceProcedure(); - internal readonly static ReadResourceProcedure ReadResource = new ReadResourceProcedure(); - internal readonly static SelectCurrentSchemaVersionProcedure SelectCurrentSchemaVersion = new SelectCurrentSchemaVersionProcedure(); - internal readonly static UpsertResourceProcedure UpsertResource = new UpsertResourceProcedure(); - internal readonly static UpsertSchemaVersionProcedure UpsertSchemaVersion = new UpsertSchemaVersionProcedure(); internal readonly static ClaimTypeTable ClaimType = new ClaimTypeTable(); + internal readonly static CompartmentAssignmentTable CompartmentAssignment = new CompartmentAssignmentTable(); + internal readonly static CompartmentTypeTable CompartmentType = new CompartmentTypeTable(); internal readonly static QuantityCodeTable QuantityCode = new QuantityCodeTable(); internal readonly static ResourceTable Resource = new ResourceTable(); internal readonly static ResourceTypeTable ResourceType = new ResourceTypeTable(); @@ -23,6 +20,121 @@ internal class V1 internal readonly static SchemaVersionTable SchemaVersion = new SchemaVersionTable(); internal readonly static SearchParamTable SearchParam = new SearchParamTable(); internal readonly static SystemTable System = new SystemTable(); + internal readonly static HardDeleteResourceProcedure HardDeleteResource = new HardDeleteResourceProcedure(); + internal readonly static ReadResourceProcedure ReadResource = new ReadResourceProcedure(); + internal readonly static SelectCurrentSchemaVersionProcedure SelectCurrentSchemaVersion = new SelectCurrentSchemaVersionProcedure(); + internal readonly static UpsertResourceProcedure UpsertResource = new UpsertResourceProcedure(); + internal readonly static UpsertSchemaVersionProcedure UpsertSchemaVersion = new UpsertSchemaVersionProcedure(); + internal class ClaimTypeTable : Table + { + internal ClaimTypeTable(): base("dbo.ClaimType") + { + } + + internal readonly TinyIntColumn ClaimTypeId = new TinyIntColumn("ClaimTypeId"); + internal readonly VarCharColumn Name = new VarCharColumn("Name", 128); + } + + internal class CompartmentAssignmentTable : Table + { + internal CompartmentAssignmentTable(): base("dbo.CompartmentAssignment") + { + } + + internal readonly BigIntColumn ResourceSurrogateId = new BigIntColumn("ResourceSurrogateId"); + internal readonly TinyIntColumn CompartmentTypeId = new TinyIntColumn("CompartmentTypeId"); + internal readonly VarCharColumn ReferenceResourceId = new VarCharColumn("ReferenceResourceId", 64); + internal readonly BitColumn IsHistory = new BitColumn("IsHistory"); + } + + internal class CompartmentTypeTable : Table + { + internal CompartmentTypeTable(): base("dbo.CompartmentType") + { + } + + internal readonly TinyIntColumn CompartmentTypeId = new TinyIntColumn("CompartmentTypeId"); + internal readonly VarCharColumn Name = new VarCharColumn("Name", 128); + } + + internal class QuantityCodeTable : Table + { + internal QuantityCodeTable(): base("dbo.QuantityCode") + { + } + + internal readonly IntColumn QuantityCodeId = new IntColumn("QuantityCodeId"); + internal readonly NVarCharColumn Value = new NVarCharColumn("Value", 256); + } + + internal class ResourceTable : Table + { + internal ResourceTable(): base("dbo.Resource") + { + } + + internal readonly SmallIntColumn ResourceTypeId = new SmallIntColumn("ResourceTypeId"); + internal readonly VarCharColumn ResourceId = new VarCharColumn("ResourceId", 64); + internal readonly IntColumn Version = new IntColumn("Version"); + internal readonly BitColumn IsHistory = new BitColumn("IsHistory"); + internal readonly BigIntColumn ResourceSurrogateId = new BigIntColumn("ResourceSurrogateId"); + internal readonly DateTime2Column LastUpdated = new DateTime2Column("LastUpdated", 7); + internal readonly BitColumn IsDeleted = new BitColumn("IsDeleted"); + internal readonly NullableVarCharColumn RequestMethod = new NullableVarCharColumn("RequestMethod", 10); + internal readonly VarBinaryColumn RawResource = new VarBinaryColumn("RawResource", -1); + } + + internal class ResourceTypeTable : Table + { + internal ResourceTypeTable(): base("dbo.ResourceType") + { + } + + internal readonly SmallIntColumn ResourceTypeId = new SmallIntColumn("ResourceTypeId"); + internal readonly NVarCharColumn Name = new NVarCharColumn("Name", 50); + } + + internal class ResourceWriteClaimTable : Table + { + internal ResourceWriteClaimTable(): base("dbo.ResourceWriteClaim") + { + } + + internal readonly BigIntColumn ResourceSurrogateId = new BigIntColumn("ResourceSurrogateId"); + internal readonly TinyIntColumn ClaimTypeId = new TinyIntColumn("ClaimTypeId"); + internal readonly NVarCharColumn ClaimValue = new NVarCharColumn("ClaimValue", 128); + } + + internal class SchemaVersionTable : Table + { + internal SchemaVersionTable(): base("dbo.SchemaVersion") + { + } + + internal readonly IntColumn Version = new IntColumn("Version"); + internal readonly VarCharColumn Status = new VarCharColumn("Status", 10); + } + + internal class SearchParamTable : Table + { + internal SearchParamTable(): base("dbo.SearchParam") + { + } + + internal readonly SmallIntColumn SearchParamId = new SmallIntColumn("SearchParamId"); + internal readonly VarCharColumn Uri = new VarCharColumn("Uri", 128); + } + + internal class SystemTable : Table + { + internal SystemTable(): base("dbo.System") + { + } + + internal readonly IntColumn SystemId = new IntColumn("SystemId"); + internal readonly NVarCharColumn Value = new NVarCharColumn("Value", 256); + } + internal class HardDeleteResourceProcedure : StoredProcedure { internal HardDeleteResourceProcedure(): base("dbo.HardDeleteResource") @@ -88,7 +200,8 @@ internal UpsertResourceProcedure(): base("dbo.UpsertResource") private readonly ParameterDefinition _requestMethod = new ParameterDefinition("@requestMethod", global::System.Data.SqlDbType.VarChar, false, 10); private readonly ParameterDefinition _rawResource = new ParameterDefinition("@rawResource", global::System.Data.SqlDbType.VarBinary, false, -1); private readonly ResourceWriteClaimTableTypeTableValuedParameterDefinition _resourceWriteClaims = new ResourceWriteClaimTableTypeTableValuedParameterDefinition("@resourceWriteClaims"); - public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, System.Int16 resourceTypeId, System.String resourceId, System.Nullable eTag, System.Boolean allowCreate, System.Boolean isDeleted, System.DateTimeOffset updatedDateTime, System.Boolean keepHistory, System.String requestMethod, global::System.IO.Stream rawResource, global::System.Collections.Generic.IEnumerable resourceWriteClaims) + private readonly CompartmentAssignmentTableTypeTableValuedParameterDefinition _compartmentAssignments = new CompartmentAssignmentTableTypeTableValuedParameterDefinition("@compartmentAssignments"); + public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, System.Int16 resourceTypeId, System.String resourceId, System.Nullable eTag, System.Boolean allowCreate, System.Boolean isDeleted, System.DateTimeOffset updatedDateTime, System.Boolean keepHistory, System.String requestMethod, global::System.IO.Stream rawResource, global::System.Collections.Generic.IEnumerable resourceWriteClaims, global::System.Collections.Generic.IEnumerable compartmentAssignments) { command.CommandType = global::System.Data.CommandType.StoredProcedure; command.CommandText = "dbo.UpsertResource"; @@ -102,117 +215,105 @@ public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, Sy _requestMethod.AddParameter(command.Parameters, requestMethod); _rawResource.AddParameter(command.Parameters, rawResource); _resourceWriteClaims.AddParameter(command.Parameters, resourceWriteClaims); + _compartmentAssignments.AddParameter(command.Parameters, compartmentAssignments); + } + + public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, System.Int16 resourceTypeId, System.String resourceId, System.Nullable eTag, System.Boolean allowCreate, System.Boolean isDeleted, System.DateTimeOffset updatedDateTime, System.Boolean keepHistory, System.String requestMethod, global::System.IO.Stream rawResource, UpsertResourceTableValuedParameters tableValuedParameters) + { + PopulateCommand(command, resourceTypeId: resourceTypeId, resourceId: resourceId, eTag: eTag, allowCreate: allowCreate, isDeleted: isDeleted, updatedDateTime: updatedDateTime, keepHistory: keepHistory, requestMethod: requestMethod, rawResource: rawResource, resourceWriteClaims: tableValuedParameters.ResourceWriteClaims, compartmentAssignments: tableValuedParameters.CompartmentAssignments); } } - internal class UpsertSchemaVersionProcedure : StoredProcedure + internal class UpsertResourceTvpGenerator : IStoredProcedureTableValuedParametersGenerator { - internal UpsertSchemaVersionProcedure(): base("dbo.UpsertSchemaVersion") + public UpsertResourceTvpGenerator(ITableValuedParameterRowGenerator ResourceWriteClaimTableTypeRowGenerator, ITableValuedParameterRowGenerator CompartmentAssignmentTableTypeRowGenerator) { + this.ResourceWriteClaimTableTypeRowGenerator = ResourceWriteClaimTableTypeRowGenerator; + this.CompartmentAssignmentTableTypeRowGenerator = CompartmentAssignmentTableTypeRowGenerator; } - private readonly ParameterDefinition _version = new ParameterDefinition("@version", global::System.Data.SqlDbType.Int, false); - private readonly ParameterDefinition _status = new ParameterDefinition("@status", global::System.Data.SqlDbType.VarChar, false, 10); - public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, System.Int32 version, System.String status) + private readonly ITableValuedParameterRowGenerator ResourceWriteClaimTableTypeRowGenerator; + private readonly ITableValuedParameterRowGenerator CompartmentAssignmentTableTypeRowGenerator; + public UpsertResourceTableValuedParameters Generate(TInput input) { - command.CommandType = global::System.Data.CommandType.StoredProcedure; - command.CommandText = "dbo.UpsertSchemaVersion"; - _version.AddParameter(command.Parameters, version); - _status.AddParameter(command.Parameters, status); + return new UpsertResourceTableValuedParameters(ResourceWriteClaimTableTypeRowGenerator.GenerateRows(input), CompartmentAssignmentTableTypeRowGenerator.GenerateRows(input)); } } - internal class ClaimTypeTable : Table + internal struct UpsertResourceTableValuedParameters { - internal ClaimTypeTable(): base("dbo.ClaimType") + internal UpsertResourceTableValuedParameters(global::System.Collections.Generic.IEnumerable ResourceWriteClaims, global::System.Collections.Generic.IEnumerable CompartmentAssignments) { + this.ResourceWriteClaims = ResourceWriteClaims; + this.CompartmentAssignments = CompartmentAssignments; } - internal readonly TinyIntColumn ClaimTypeId = new TinyIntColumn("ClaimTypeId"); - internal readonly VarCharColumn Name = new VarCharColumn("Name", 128); - } - - internal class QuantityCodeTable : Table - { - internal QuantityCodeTable(): base("dbo.QuantityCode") + internal global::System.Collections.Generic.IEnumerable ResourceWriteClaims { + get; } - internal readonly IntColumn QuantityCodeId = new IntColumn("QuantityCodeId"); - internal readonly NVarCharColumn Value = new NVarCharColumn("Value", 256); - } - - internal class ResourceTable : Table - { - internal ResourceTable(): base("dbo.Resource") + internal global::System.Collections.Generic.IEnumerable CompartmentAssignments { + get; } - - internal readonly SmallIntColumn ResourceTypeId = new SmallIntColumn("ResourceTypeId"); - internal readonly VarCharColumn ResourceId = new VarCharColumn("ResourceId", 64); - internal readonly IntColumn Version = new IntColumn("Version"); - internal readonly BitColumn IsHistory = new BitColumn("IsHistory"); - internal readonly BigIntColumn ResourceSurrogateId = new BigIntColumn("ResourceSurrogateId"); - internal readonly DateTime2Column LastUpdated = new DateTime2Column("LastUpdated", 7); - internal readonly BitColumn IsDeleted = new BitColumn("IsDeleted"); - internal readonly NullableVarCharColumn RequestMethod = new NullableVarCharColumn("RequestMethod", 10); - internal readonly VarBinaryColumn RawResource = new VarBinaryColumn("RawResource", -1); } - internal class ResourceTypeTable : Table + internal class UpsertSchemaVersionProcedure : StoredProcedure { - internal ResourceTypeTable(): base("dbo.ResourceType") + internal UpsertSchemaVersionProcedure(): base("dbo.UpsertSchemaVersion") { } - internal readonly SmallIntColumn ResourceTypeId = new SmallIntColumn("ResourceTypeId"); - internal readonly NVarCharColumn Name = new NVarCharColumn("Name", 50); - } - - internal class ResourceWriteClaimTable : Table - { - internal ResourceWriteClaimTable(): base("dbo.ResourceWriteClaim") + private readonly ParameterDefinition _version = new ParameterDefinition("@version", global::System.Data.SqlDbType.Int, false); + private readonly ParameterDefinition _status = new ParameterDefinition("@status", global::System.Data.SqlDbType.VarChar, false, 10); + public void PopulateCommand(global::System.Data.SqlClient.SqlCommand command, System.Int32 version, System.String status) { + command.CommandType = global::System.Data.CommandType.StoredProcedure; + command.CommandText = "dbo.UpsertSchemaVersion"; + _version.AddParameter(command.Parameters, version); + _status.AddParameter(command.Parameters, status); } - - internal readonly BigIntColumn ResourceSurrogateId = new BigIntColumn("ResourceSurrogateId"); - internal readonly TinyIntColumn ClaimTypeId = new TinyIntColumn("ClaimTypeId"); - internal readonly NVarCharColumn ClaimValue = new NVarCharColumn("ClaimValue", 128); } - internal class SchemaVersionTable : Table + private class CompartmentAssignmentTableTypeTableValuedParameterDefinition : TableValuedParameterDefinition { - internal SchemaVersionTable(): base("dbo.SchemaVersion") + internal CompartmentAssignmentTableTypeTableValuedParameterDefinition(System.String parameterName): base(parameterName, "dbo.CompartmentAssignmentTableType_1") { } - internal readonly IntColumn Version = new IntColumn("Version"); - internal readonly VarCharColumn Status = new VarCharColumn("Status", 10); + internal readonly TinyIntColumn CompartmentTypeId = new TinyIntColumn("CompartmentTypeId"); + internal readonly VarCharColumn ReferenceResourceId = new VarCharColumn("ReferenceResourceId", 64); + protected override global::System.Collections.Generic.IEnumerable Columns => new Column[]{CompartmentTypeId, ReferenceResourceId}; + protected override void FillSqlDataRecord(global::Microsoft.SqlServer.Server.SqlDataRecord record, CompartmentAssignmentTableTypeRow rowData) + { + CompartmentTypeId.Set(record, 0, rowData.CompartmentTypeId); + ReferenceResourceId.Set(record, 1, rowData.ReferenceResourceId); + } } - internal class SearchParamTable : Table + internal struct CompartmentAssignmentTableTypeRow { - internal SearchParamTable(): base("dbo.SearchParam") + internal CompartmentAssignmentTableTypeRow(System.Byte CompartmentTypeId, System.String ReferenceResourceId) { + this.CompartmentTypeId = CompartmentTypeId; + this.ReferenceResourceId = ReferenceResourceId; } - internal readonly SmallIntColumn SearchParamId = new SmallIntColumn("SearchParamId"); - internal readonly VarCharColumn Uri = new VarCharColumn("Uri", 128); - } - - internal class SystemTable : Table - { - internal SystemTable(): base("dbo.System") + internal System.Byte CompartmentTypeId { + get; } - internal readonly IntColumn SystemId = new IntColumn("SystemId"); - internal readonly NVarCharColumn Value = new NVarCharColumn("Value", 256); + internal System.String ReferenceResourceId + { + get; + } } private class ResourceWriteClaimTableTypeTableValuedParameterDefinition : TableValuedParameterDefinition { - internal ResourceWriteClaimTableTypeTableValuedParameterDefinition(System.String parameterName): base(parameterName, "dbo.ResourceWriteClaimTableType") + internal ResourceWriteClaimTableTypeTableValuedParameterDefinition(System.String parameterName): base(parameterName, "dbo.ResourceWriteClaimTableType_1") { } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 548611ca48..7952cdb0bc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -4,13 +4,11 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Collections.Generic; using System.Data; using System.Data.SqlClient; using System.Globalization; using System.IO; using System.IO.Compression; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -21,7 +19,6 @@ using Microsoft.Health.Fhir.Core; using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features.Conformance; -using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.SqlServer.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; @@ -33,23 +30,29 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage /// /// A SQL Server-backed . /// - public class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability + internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability { private static readonly Encoding ResourceEncoding = new UnicodeEncoding(bigEndian: false, byteOrderMark: false); private readonly SqlServerDataStoreConfiguration _configuration; private readonly SqlServerFhirModel _model; + private readonly V1.UpsertResourceTvpGenerator _upsertResourceTvpGenerator; private readonly RecyclableMemoryStreamManager _memoryStreamManager; private readonly ILogger _logger; - public SqlServerFhirDataStore(SqlServerDataStoreConfiguration configuration, SqlServerFhirModel model, ILogger logger) + public SqlServerFhirDataStore( + SqlServerDataStoreConfiguration configuration, + SqlServerFhirModel model, + V1.UpsertResourceTvpGenerator upsertResourceTvpGenerator, + ILogger logger) { EnsureArg.IsNotNull(configuration, nameof(configuration)); EnsureArg.IsNotNull(model, nameof(model)); + EnsureArg.IsNotNull(upsertResourceTvpGenerator, nameof(upsertResourceTvpGenerator)); EnsureArg.IsNotNull(logger, nameof(logger)); - _configuration = configuration; _model = model; + _upsertResourceTvpGenerator = upsertResourceTvpGenerator; _logger = logger; _memoryStreamManager = new RecyclableMemoryStreamManager(); } @@ -69,14 +72,14 @@ public async Task UpsertAsync(ResourceWrapper resource, WeakETag await connection.OpenAsync(cancellationToken); using (var command = connection.CreateCommand()) - using (var ms = new RecyclableMemoryStream(_memoryStreamManager)) - using (var gzipStream = new GZipStream(ms, CompressionMode.Compress)) + using (var stream = new RecyclableMemoryStream(_memoryStreamManager)) + using (var gzipStream = new GZipStream(stream, CompressionMode.Compress)) using (var writer = new StreamWriter(gzipStream, ResourceEncoding)) { writer.Write(resource.RawResource.Data); writer.Flush(); - ms.Seek(0, 0); + stream.Seek(0, 0); V1.UpsertResource.PopulateCommand( command, @@ -88,8 +91,8 @@ public async Task UpsertAsync(ResourceWrapper resource, WeakETag updatedDateTime: resource.LastModified, keepHistory: keepHistory, requestMethod: resource.Request.Method, - rawResource: ms, - resourceWriteClaims: GetResourceWriteClaims(resource)); + rawResource: stream, + tableValuedParameters: _upsertResourceTvpGenerator.Generate(resource)); try { @@ -121,11 +124,6 @@ public async Task UpsertAsync(ResourceWrapper resource, WeakETag } } - private IEnumerable GetResourceWriteClaims(ResourceWrapper resource) - { - return resource.LastModifiedClaims?.Select(c => new V1.ResourceWriteClaimTableTypeRow(_model.GetClaimTypeId(c.Key), c.Value)); - } - public async Task GetAsync(ResourceKey key, CancellationToken cancellationToken) { await _model.EnsureInitialized(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirModel.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirModel.cs index 5e180b30b1..e8f5627491 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirModel.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirModel.cs @@ -39,14 +39,14 @@ public sealed class SqlServerFhirModel : IDisposable private readonly ILogger _logger; private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; private readonly SecurityConfiguration _securityConfiguration; + private readonly RetryableInitializationOperation _initializationOperation; private Dictionary _resourceTypeToId; private Dictionary _resourceTypeIdToTypeName; private Dictionary _searchParamUriToId; private ConcurrentDictionary _systemToId; private ConcurrentDictionary _quantityCodeToId; private Dictionary _claimNameToId; - - private readonly RetryableInitializationOperation _initializationOperation; + private Dictionary _compartmentTypeToId; public SqlServerFhirModel( SqlServerDataStoreConfiguration configuration, @@ -93,6 +93,12 @@ public short GetSearchParamId(string searchParamUri) return _searchParamUriToId[searchParamUri]; } + public byte GetCompartmentId(CompartmentType compartmentType) + { + ThrowIfNotInitialized(); + return _compartmentTypeToId[compartmentType]; + } + public int GetSystem(string system) { ThrowIfNotInitialized(); @@ -150,22 +156,31 @@ SELECT value FROM string_split(@claimTypes, ',') -- result set 3 SELECT ClaimTypeId, Name FROM dbo.ClaimType; + + INSERT INTO dbo.CompartmentType (Name) + SELECT value FROM string_split(@compartmentTypes, ',') + EXCEPT SELECT Name FROM dbo.CompartmentType; + + -- result set 4 + SELECT CompartmentTypeId, Name FROM dbo.CompartmentType; COMMIT TRANSACTION - -- result set 4 + -- result set 5 SELECT Value, SystemId from dbo.System; - -- result set 5 + -- result set 6 SELECT Value, QuantityCodeId FROM dbo.QuantityCode"; string commaSeparatedResourceTypes = string.Join(",", ModelInfo.SupportedResources); string searchParametersJson = JsonConvert.SerializeObject(_searchParameterDefinitionManager.AllSearchParameters.Select(p => new { Name = p.Name, Uri = p.Url })); string commaSeparatedClaimTypes = string.Join(',', _securityConfiguration.LastModifiedClaims); + string commaSeparatedCompartmentTypes = string.Join(',', Enum.GetNames(typeof(CompartmentType))); sqlCommand.Parameters.AddWithValue("@resourceTypes", commaSeparatedResourceTypes); sqlCommand.Parameters.AddWithValue("@searchParams", searchParametersJson); sqlCommand.Parameters.AddWithValue("@claimTypes", commaSeparatedClaimTypes); + sqlCommand.Parameters.AddWithValue("@compartmentTypes", commaSeparatedCompartmentTypes); using (SqlDataReader reader = await sqlCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess)) { @@ -175,6 +190,7 @@ COMMIT TRANSACTION var systemToId = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); var quantityCodeToId = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); var claimNameToId = new Dictionary(StringComparer.Ordinal); + var compartmentTypeToId = new Dictionary(); // result set 1 while (reader.Read()) @@ -206,13 +222,22 @@ COMMIT TRANSACTION // result set 4 reader.NextResult(); + while (reader.Read()) + { + (byte id, string compartmentName) = reader.ReadRow(V1.CompartmentType.CompartmentTypeId, V1.CompartmentType.Name); + compartmentTypeToId.Add(Enum.Parse(compartmentName), id); + } + + // result set 5 + reader.NextResult(); + while (reader.Read()) { var (value, systemId) = reader.ReadRow(V1.System.Value, V1.System.SystemId); systemToId.TryAdd(value, systemId); } - // result set 5 + // result set 6 reader.NextResult(); while (reader.Read()) @@ -227,6 +252,7 @@ COMMIT TRANSACTION _systemToId = systemToId; _quantityCodeToId = quantityCodeToId; _claimNameToId = claimNameToId; + _compartmentTypeToId = compartmentTypeToId; } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/CompartmentAssignmentRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/CompartmentAssignmentRowGenerator.cs new file mode 100644 index 0000000000..54753f03c3 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/CompartmentAssignmentRowGenerator.cs @@ -0,0 +1,98 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Threading; +using EnsureThat; +using Hl7.Fhir.Model; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration +{ + internal class CompartmentAssignmentRowGenerator : ITableValuedParameterRowGenerator + { + private readonly SqlServerFhirModel _model; + private bool _initialized; + private byte _patientCompartmentId; + private byte _encounterCompartmentId; + private byte _relatedPersonCompartmentId; + private byte _practitionerCompartmentId; + private byte _deviceCompartmentId; + + public CompartmentAssignmentRowGenerator(SqlServerFhirModel model) + { + EnsureArg.IsNotNull(model, nameof(model)); + _model = model; + } + + public IEnumerable GenerateRows(ResourceWrapper resource) + { + EnsureInitialized(); + + var compartments = resource.CompartmentIndices; + if (compartments == null) + { + yield break; + } + + if (compartments.PatientCompartmentEntry != null) + { + foreach (var entry in compartments.PatientCompartmentEntry) + { + yield return new V1.CompartmentAssignmentTableTypeRow(_patientCompartmentId, entry); + } + } + + if (compartments.EncounterCompartmentEntry != null) + { + foreach (var entry in compartments.EncounterCompartmentEntry) + { + yield return new V1.CompartmentAssignmentTableTypeRow(_encounterCompartmentId, entry); + } + } + + if (compartments.RelatedPersonCompartmentEntry != null) + { + foreach (var entry in compartments.RelatedPersonCompartmentEntry) + { + yield return new V1.CompartmentAssignmentTableTypeRow(_relatedPersonCompartmentId, entry); + } + } + + if (compartments.PractitionerCompartmentEntry != null) + { + foreach (var entry in compartments.PractitionerCompartmentEntry) + { + yield return new V1.CompartmentAssignmentTableTypeRow(_practitionerCompartmentId, entry); + } + } + + if (compartments.DeviceCompartmentEntry != null) + { + foreach (var entry in compartments.DeviceCompartmentEntry) + { + yield return new V1.CompartmentAssignmentTableTypeRow(_deviceCompartmentId, entry); + } + } + } + + private void EnsureInitialized() + { + if (Volatile.Read(ref _initialized)) + { + return; + } + + _patientCompartmentId = _model.GetCompartmentId(CompartmentType.Patient); + _encounterCompartmentId = _model.GetCompartmentId(CompartmentType.Encounter); + _relatedPersonCompartmentId = _model.GetCompartmentId(CompartmentType.RelatedPerson); + _practitionerCompartmentId = _model.GetCompartmentId(CompartmentType.Practitioner); + _deviceCompartmentId = _model.GetCompartmentId(CompartmentType.Device); + + Volatile.Write(ref _initialized, true); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/ResourceWriteClaimRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/ResourceWriteClaimRowGenerator.cs new file mode 100644 index 0000000000..859a931ddc --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/ResourceWriteClaimRowGenerator.cs @@ -0,0 +1,30 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration +{ + internal class ResourceWriteClaimRowGenerator : ITableValuedParameterRowGenerator + { + private readonly SqlServerFhirModel _model; + + public ResourceWriteClaimRowGenerator(SqlServerFhirModel model) + { + EnsureArg.IsNotNull(model, nameof(model)); + _model = model; + } + + public IEnumerable GenerateRows(ResourceWrapper resource) + { + return resource.LastModifiedClaims?.Select(c => + new V1.ResourceWriteClaimTableTypeRow(_model.GetClaimTypeId(c.Key), c.Value)); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index 8e2c44db7a..7c919c6bbd 100644 --- a/test/Microsoft.Health.Fhir.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -7,6 +7,7 @@ using System.Data.SqlClient; using System.Numerics; using Hl7.Fhir.Model; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Health.Fhir.Core.Configs; @@ -14,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.SqlServer.Configs; using Microsoft.Health.Fhir.SqlServer.Features.Schema; +using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using NSubstitute; @@ -59,7 +61,15 @@ public SqlServerFhirStorageTestsFixture() var sqlServerFhirModel = new SqlServerFhirModel(config, schemaInformation, searchParameterDefinitionManager, Options.Create(securityConfiguration), NullLogger.Instance); - _fhirDataStore = new SqlServerFhirDataStore(config, sqlServerFhirModel, NullLogger.Instance); + var serviceCollection = new ServiceCollection(); + serviceCollection.AddSqlServerTableRowParameterGenerators(); + serviceCollection.AddSingleton(sqlServerFhirModel); + + ServiceProvider serviceProvider = serviceCollection.BuildServiceProvider(); + + var upsertResourceTvpGenerator = serviceProvider.GetRequiredService>(); + + _fhirDataStore = new SqlServerFhirDataStore(config, sqlServerFhirModel, upsertResourceTvpGenerator, NullLogger.Instance); _testHelper = new SqlServerFhirStorageTestHelper(TestConnectionString); } diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs index f0ea0a94e6..da64a9386f 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateProcedureVisitor.cs @@ -8,6 +8,7 @@ using System.Data; using System.Data.SqlClient; using System.Linq; +using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.SqlServer.TransactSql.ScriptDom; @@ -20,6 +21,12 @@ namespace Microsoft.Health.Extensions.BuildTimeCodeGenerator.Sql /// internal class CreateProcedureVisitor : SqlVisitor { + private const string TvpGeneratorGenericTypeName = "TInput"; + private const string PopulateCommandMethodName = "PopulateCommand"; + private const string CommandParameterName = "command"; + + public override int ArtifactSortOder => 1; + public override void Visit(CreateProcedureStatement node) { string procedureName = node.ProcedureReference.Name.BaseIdentifier.Value; @@ -59,12 +66,16 @@ public override void Visit(CreateProcedureStatement node) .AddMembers(node.Parameters.Select(CreateFieldForParameter).ToArray()) // add the PopulateCommand method - .AddMembers(AddPopulateCommandMethod(node, schemaQualifiedProcedureName)); + .AddMembers(AddPopulateCommandMethod(node, schemaQualifiedProcedureName), AddPopulateCommandMethodForTableValuedParameters(node, procedureName)); FieldDeclarationSyntax fieldDeclarationSyntax = CreateStaticFieldForClass(className, procedureName); - MembersToAdd.Add(classDeclarationSyntax); - MembersToAdd.Add(fieldDeclarationSyntax); + var (tvpGeneratorClass, tvpHolderStruct) = CreateTvpGeneratorTypes(node, procedureName); + + MembersToAdd.Add(classDeclarationSyntax.AddSortingKey(this, procedureName)); + MembersToAdd.Add(fieldDeclarationSyntax.AddSortingKey(this, procedureName)); + MembersToAdd.Add(tvpGeneratorClass.AddSortingKey(this, procedureName)); + MembersToAdd.Add(tvpHolderStruct.AddSortingKey(this, procedureName)); base.Visit(node); } @@ -81,7 +92,7 @@ private MemberDeclarationSyntax CreateFieldForParameter(ProcedureParameter param { Argument(LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(parameter.VariableName.Value))), }; - if (Enum.TryParse(parameter.DataType.Name.BaseIdentifier.Value, ignoreCase: true, out var sqlDbType)) + if (TryGetSqlDbTypeForParameter(parameter, out SqlDbType sqlDbType)) { // new ParameterDefinition("@paramName", SqlDbType.Int, nullable, maxlength,...) typeName = GenericName("ParameterDefinition") @@ -127,11 +138,11 @@ private MethodDeclarationSyntax AddPopulateCommandMethod(CreateProcedureStatemen { return MethodDeclaration( typeof(void).ToTypeSyntax(), - Identifier("PopulateCommand")) + Identifier(PopulateCommandMethodName)) .AddModifiers(Token(SyntaxKind.PublicKeyword)) // first parameter is the SqlCommand - .AddParameterListParameters(Parameter(Identifier("command")).WithType(typeof(SqlCommand).ToTypeSyntax(useGlobalAlias: true))) + .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(typeof(SqlCommand).ToTypeSyntax(useGlobalAlias: true))) // Add a parameter for each stored procedure parameter .AddParameterListParameters(node.Parameters.Select(selector: p => @@ -147,7 +158,7 @@ private MethodDeclarationSyntax AddPopulateCommandMethod(CreateProcedureStatemen SyntaxKind.SimpleAssignmentExpression, MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("command"), + IdentifierName(CommandParameterName), IdentifierName("CommandType")), MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, @@ -158,7 +169,7 @@ private MethodDeclarationSyntax AddPopulateCommandMethod(CreateProcedureStatemen SyntaxKind.SimpleAssignmentExpression, MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("command"), + IdentifierName(CommandParameterName), IdentifierName("CommandText")), LiteralExpression(SyntaxKind.StringLiteralExpression, Literal(schemaQualifiedProcedureName))))) @@ -173,11 +184,205 @@ private MethodDeclarationSyntax AddPopulateCommandMethod(CreateProcedureStatemen .AddArgumentListArguments( Argument(MemberAccessExpression( SyntaxKind.SimpleMemberAccessExpression, - IdentifierName("command"), + IdentifierName(CommandParameterName), IdentifierName("Parameters"))), Argument(IdentifierName(ParameterNameForParameter(p)))))).ToArray()); } + private MemberDeclarationSyntax AddPopulateCommandMethodForTableValuedParameters(CreateProcedureStatement node, string procedureName) + { + var nonTableParameters = new List(); + var tableParameters = new List(); + + foreach (var procedureParameter in node.Parameters) + { + if (TryGetSqlDbTypeForParameter(procedureParameter, out _)) + { + nonTableParameters.Add(procedureParameter); + } + else + { + tableParameters.Add(procedureParameter); + } + } + + if (tableParameters.Count == 0) + { + return IncompleteMember(); + } + + string tableValuedParametersParameterName = "tableValuedParameters"; + + return MethodDeclaration( + typeof(void).ToTypeSyntax(), + Identifier(PopulateCommandMethodName)) + .AddModifiers(Token(SyntaxKind.PublicKeyword)) + + // first parameter is the SqlCommand + .AddParameterListParameters(Parameter(Identifier(CommandParameterName)).WithType(typeof(SqlCommand).ToTypeSyntax(useGlobalAlias: true))) + + // Add a parameter for each non-TVP + .AddParameterListParameters(nonTableParameters.Select(selector: p => + Parameter(Identifier(ParameterNameForParameter(p))) + .WithType(DataTypeReferenceToClrType(p.DataType, p.Value != null))).ToArray()) + + // Add a parameter for the TVP set + .AddParameterListParameters( + Parameter(Identifier(tableValuedParametersParameterName)).WithType(IdentifierName(TableValuedParametersStructName(procedureName)))) + + // Call the overload + .AddBodyStatements( + ExpressionStatement( + InvocationExpression( + IdentifierName(PopulateCommandMethodName)) + .AddArgumentListArguments(Argument(IdentifierName(CommandParameterName))) + .AddArgumentListArguments( + nonTableParameters.Select(p => + Argument(IdentifierName(ParameterNameForParameter(p))) + .WithNameColon(NameColon(ParameterNameForParameter(p)))).ToArray()) + .AddArgumentListArguments( + tableParameters.Select(p => + Argument(MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + IdentifierName(tableValuedParametersParameterName), + IdentifierName(PropertyNameForParameter(p)))) + .WithNameColon(NameColon(ParameterNameForParameter(p)))).ToArray()))); + } + + private (MemberDeclarationSyntax tvpGeneratorClass, MemberDeclarationSyntax tvpHolderStruct) CreateTvpGeneratorTypes(CreateProcedureStatement node, string procedureName) + { + List<(string parameterName, string rowStructName)> rowTypes = node.Parameters + .Where(p => !TryGetSqlDbTypeForParameter(p, out _)) + .Select(p => (parameterName: PropertyNameForParameter(p), rowStructName: GetRowStructNameForTableType(p.DataType.Name))) + .ToList(); + + if (rowTypes.Count == 0) + { + // no table-valued parameters on this procedure + return (IncompleteMember(), IncompleteMember()); + } + + var holderStructName = TableValuedParametersStructName(procedureName); + + // create a struct with properties for each table-valued parameter + + var structDeclaration = StructDeclaration(holderStructName) + .AddModifiers(Token(SyntaxKind.InternalKeyword)) + + // Add a constructor with parameters for each column, setting the associated property for each column. + .AddMembers( + ConstructorDeclaration( + Identifier(holderStructName)) + .WithModifiers( + TokenList( + Token(SyntaxKind.InternalKeyword))) + .AddParameterListParameters( + rowTypes.Select(p => + Parameter(Identifier(p.parameterName)) + .WithType(TypeExtensions.CreateGenericTypeFromGenericTypeDefinition( + typeof(IEnumerable<>).ToTypeSyntax(true), + IdentifierName(p.rowStructName)))).ToArray()) + .WithBody( + Block(rowTypes.Select(p => + ExpressionStatement( + AssignmentExpression( + SyntaxKind.SimpleAssignmentExpression, + left: MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + ThisExpression(), + IdentifierName(p.parameterName)), + right: IdentifierName(p.parameterName))))))) + + // Add a property for each column + .AddMembers(rowTypes.Select(p => + (MemberDeclarationSyntax)PropertyDeclaration( + TypeExtensions.CreateGenericTypeFromGenericTypeDefinition( + typeof(IEnumerable<>).ToTypeSyntax(true), + IdentifierName(p.rowStructName)), + Identifier(p.parameterName)) + .AddModifiers(Token(SyntaxKind.InternalKeyword)) + .AddAccessorListAccessors(AccessorDeclaration(SyntaxKind.GetAccessorDeclaration) + .WithSemicolonToken(Token(SyntaxKind.SemicolonToken)))).ToArray()); + + string className = $"{procedureName}TvpGenerator"; + + List distinctTvpTypeNames = rowTypes.Select(r => r.rowStructName).Distinct().ToList(); + + var classDeclaration = ClassDeclaration(className) + .AddTypeParameterListParameters(TypeParameter(TvpGeneratorGenericTypeName)) + .AddBaseListTypes( + SimpleBaseType( + GenericName("IStoredProcedureTableValuedParametersGenerator") + .AddTypeArgumentListArguments( + IdentifierName(TvpGeneratorGenericTypeName), + IdentifierName(holderStructName)))) + .AddModifiers(Token(SyntaxKind.InternalKeyword)) + .AddMembers( + ConstructorDeclaration(Identifier(className)) + .WithModifiers(TokenList(Token(SyntaxKind.PublicKeyword))) + .AddParameterListParameters(distinctTvpTypeNames + .Select(t => + Parameter(Identifier(GeneratorFieldName(t))) + .WithType(GeneratorType(t))).ToArray()) + .WithBody( + Block(distinctTvpTypeNames + .Select(t => + ExpressionStatement( + AssignmentExpression( + SyntaxKind.SimpleAssignmentExpression, + left: MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + ThisExpression(), + IdentifierName(GeneratorFieldName(t))), + right: IdentifierName(GeneratorFieldName(t)))))))) + .AddMembers( + distinctTvpTypeNames + .Select(t => (MemberDeclarationSyntax)FieldDeclaration( + VariableDeclaration(GeneratorType(t)) + .AddVariables(VariableDeclarator(GeneratorFieldName(t)))) + .AddModifiers(Token(SyntaxKind.PrivateKeyword), Token(SyntaxKind.ReadOnlyKeyword))).ToArray()) + .AddMembers( + MethodDeclaration(IdentifierName(holderStructName), Identifier("Generate")) + .AddModifiers(Token(SyntaxKind.PublicKeyword)) + .AddParameterListParameters(Parameter(Identifier("input")).WithType(IdentifierName(TvpGeneratorGenericTypeName))) + .AddBodyStatements( + ReturnStatement( + ObjectCreationExpression(IdentifierName(holderStructName)) + .AddArgumentListArguments( + rowTypes.Select(p => Argument( + InvocationExpression( + MemberAccessExpression( + SyntaxKind.SimpleMemberAccessExpression, + IdentifierName(GeneratorFieldName(p.rowStructName)), + IdentifierName("GenerateRows"))) + .AddArgumentListArguments(Argument(IdentifierName("input"))))).ToArray())))); + + return (classDeclaration, structDeclaration); + } + + private static string TableValuedParametersStructName(string procedureName) + { + return $"{procedureName}TableValuedParameters"; + } + + private static string GeneratorFieldName(string tableTypeName) + { + return $"{tableTypeName}Generator"; + } + + private static TypeSyntax GeneratorType(string rowStructName) + { + return GenericName("ITableValuedParameterRowGenerator") + .AddTypeArgumentListArguments( + IdentifierName(TvpGeneratorGenericTypeName), + IdentifierName(rowStructName)); + } + + private static bool TryGetSqlDbTypeForParameter(ProcedureParameter parameter, out SqlDbType sqlDbType) + { + return Enum.TryParse(parameter.DataType.Name.BaseIdentifier.Value, ignoreCase: true, out sqlDbType); + } + private static string ParameterNameForParameter(ProcedureParameter parameter) { return parameter.VariableName.Value.Substring(1); @@ -187,5 +392,11 @@ private static string FieldNameForParameter(ProcedureParameter parameter) { return $"_{parameter.VariableName.Value.Substring(1)}"; } + + private static string PropertyNameForParameter(ProcedureParameter parameter) + { + string parameterName = parameter.VariableName.Value; + return $"{char.ToUpperInvariant(parameterName[1])}{(parameterName.Length > 2 ? parameterName.Substring(2) : string.Empty)}"; + } } } diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableTypeVisitor.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableTypeVisitor.cs index a26592a22c..98fcbd70db 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableTypeVisitor.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableTypeVisitor.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Linq; -using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.SqlServer.Server; @@ -21,6 +20,8 @@ namespace Microsoft.Health.Extensions.BuildTimeCodeGenerator.Sql /// public class CreateTableTypeVisitor : SqlVisitor { + public override int ArtifactSortOder => 2; + public override void Visit(CreateTypeTableStatement node) { string tableTypeName = node.Name.BaseIdentifier.Value; @@ -141,8 +142,8 @@ public override void Visit(CreateTypeTableStatement node) .AddAccessorListAccessors(AccessorDeclaration(SyntaxKind.GetAccessorDeclaration) .WithSemicolonToken(Token(SyntaxKind.SemicolonToken)))).ToArray()); - MembersToAdd.Add(classDeclarationSyntax); - MembersToAdd.Add(rowStruct); + MembersToAdd.Add(classDeclarationSyntax.AddSortingKey(this, tableTypeName)); + MembersToAdd.Add(rowStruct.AddSortingKey(this, tableTypeName)); base.Visit(node); } diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableVisitor.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableVisitor.cs index 1ef87de680..769cca4e29 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableVisitor.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/CreateTableVisitor.cs @@ -3,9 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; -using System.Data; using System.Linq; +using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.SqlServer.TransactSql.ScriptDom; @@ -18,6 +17,8 @@ namespace Microsoft.Health.Extensions.BuildTimeCodeGenerator.Sql /// internal class CreateTableVisitor : SqlVisitor { + public override int ArtifactSortOder => 0; + public override void Visit(CreateTableStatement node) { string tableName = node.SchemaObjectName.BaseIdentifier.Value; @@ -52,8 +53,8 @@ public override void Visit(CreateTableStatement node) FieldDeclarationSyntax field = CreateStaticFieldForClass(className, tableName); - MembersToAdd.Add(classDeclarationSyntax); - MembersToAdd.Add(field); + MembersToAdd.Add(field.AddSortingKey(this, tableName)); + MembersToAdd.Add(classDeclarationSyntax.AddSortingKey(this, tableName)); base.Visit(node); } diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/MemberSorting.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/MemberSorting.cs new file mode 100644 index 0000000000..97abfe1a62 --- /dev/null +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/MemberSorting.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +namespace Microsoft.Health.Extensions.BuildTimeCodeGenerator.Sql +{ + public static class MemberSorting + { + private const string SortKey = "SortKey"; + + public static readonly Comparer Comparer = Comparer.Create(CompareMembers); + + private static int CompareMembers(MemberDeclarationSyntax a, MemberDeclarationSyntax b) + { + string[] GetSortKey(MemberDeclarationSyntax member) + { + return member.GetAnnotations(SortKey).SingleOrDefault()?.Data.Split(':') ?? throw new InvalidOperationException("Members are required to have a sort key"); + } + + if (a == b) + { + return 0; + } + + if (a == null) + { + return -1; + } + + if (b == null) + { + return 1; + } + + return GetSortKey(a).Zip(GetSortKey(b), (ta, tb) => (tokenA: ta, tokenB: tb)).Aggregate(0, (acc, curr) => acc != 0 ? acc : string.CompareOrdinal(curr.tokenA, curr.tokenB)); + } + + public static TMember AddSortingKey(this TMember member, SqlVisitor visitor, string name) + where TMember : MemberDeclarationSyntax + { + return member.WithAdditionalAnnotations(new SyntaxAnnotation(SortKey, $"{(member is FieldDeclarationSyntax ? 0 : 1)}:{visitor.ArtifactSortOder}:{name}")); + } + } +} diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs index 9e3269ed2a..d028c03440 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlModelGenerator.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.IO; using System.Linq; using EnsureThat; @@ -40,60 +41,12 @@ public SqlModelGenerator(string[] args) sqlFragment.Accept(sqlVisitor); } - var members = visitors - .SelectMany(v => v.MembersToAdd) - - // Sort the members so that the order is deterministic and the file does not change randomly - // Put the fields first, followed by classes, then structs. - .OrderBy(m => m is FieldDeclarationSyntax ? 0 : m is ClassDeclarationSyntax ? 1 : 2) - .ThenBy(m => - { - switch (m) - { - case FieldDeclarationSyntax f: - - // order by the class suffix (Table, Procedure) - string fieldTypeName = f.Declaration.Type.ToString(); - string variableName = f.Declaration.Variables.First().Identifier.Text; - - return fieldTypeName.Substring(variableName.Length); - - case ClassDeclarationSyntax c: - - // order by the base type (Table, Procedure, TableValuedParameterDefinition) - BaseTypeSyntax baseType = c.BaseList?.Types.FirstOrDefault(); - if (baseType == null) - { - return string.Empty; - } - - return baseType.ToString(); - case StructDeclarationSyntax s: - return s.Identifier.ToString(); - default: - throw new NotSupportedException(m.GetType().Name); - } - }) - - // Finally order by type name. - .ThenBy(m => - { - switch (m) - { - case FieldDeclarationSyntax f: - return f.Declaration.Type.ToString(); - case ClassDeclarationSyntax c: - return c.Identifier.ToString(); - case StructDeclarationSyntax _: - return string.Empty; - default: - throw new NotSupportedException(m.GetType().Name); - } - }); - var classDeclaration = ClassDeclaration(typeName) .WithModifiers(TokenList(Token(SyntaxKind.InternalKeyword))) - .AddMembers(members.ToArray()); + .AddMembers(visitors + .SelectMany(v => v.MembersToAdd) + .OrderBy(m => m, MemberSorting.Comparer) + .ToArray()); return (classDeclaration, new UsingDirectiveSyntax[0]); } diff --git a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlVisitor.cs b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlVisitor.cs index 169f2065d3..264760d853 100644 --- a/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlVisitor.cs +++ b/tools/Microsoft.Health.Extensions.BuildTimeCodeGenerator/Sql/SqlVisitor.cs @@ -8,7 +8,7 @@ using System.Data; using System.IO; using System.Linq; -using Microsoft.CodeAnalysis; +using System.Text.RegularExpressions; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.SqlServer.TransactSql.ScriptDom; @@ -25,6 +25,8 @@ public abstract class SqlVisitor : TSqlFragmentVisitor /// public List MembersToAdd { get; } = new List(); + public abstract int ArtifactSortOder { get; } + /// /// Converts a to a /// @@ -183,7 +185,7 @@ protected static IEnumerable GetDataTypeSpecificConstructorArgum /// The class name protected static string GetClassNameForTableValuedParameterDefinition(SchemaObjectName objectName) { - return $"{objectName.BaseIdentifier.Value}TableValuedParameterDefinition"; + return $"{GetTableTypeNameWithoutVersionSuffix(objectName)}TableValuedParameterDefinition"; } /// @@ -193,7 +195,17 @@ protected static string GetClassNameForTableValuedParameterDefinition(SchemaObje /// The struct name protected static string GetRowStructNameForTableType(SchemaObjectName objectName) { - return $"{objectName.BaseIdentifier.Value}Row"; + return $"{GetTableTypeNameWithoutVersionSuffix(objectName)}Row"; + } + + /// + /// Strips away the version suffix from a table type name. + /// + /// The table type name + /// The name + private static string GetTableTypeNameWithoutVersionSuffix(SchemaObjectName objectName) + { + return Regex.Replace(objectName.BaseIdentifier.Value, @"_\d+", string.Empty); } protected MemberDeclarationSyntax CreatePropertyForColumn(ColumnDefinition column)