diff --git a/data/Templates/Hl7v2/ADT_A01.liquid b/data/Templates/Hl7v2/ADT_A01.liquid index 91e0fa7e4..aa99399e8 100644 --- a/data/Templates/Hl7v2/ADT_A01.liquid +++ b/data/Templates/Hl7v2/ADT_A01.liquid @@ -1,8 +1,8 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ - {% assign firstSegments = hl7v2Data | get_first_segments: 'PID|PD1|PV1|PV2|PR1|AVR|MSH' -%} + {% assign firstSegments = hl7v2Data | get_first_segments: 'PID|PD1|PV1|PV2|AVR|MSH' -%} {% evaluate messageHeaderId using 'ID/MessageHeader' MSH: firstSegments.MSH -%} {% if messageHeaderId -%} @@ -15,7 +15,7 @@ {% include 'Resource/Patient' PID: firstSegments.PID, PD1: firstSegments.PD1, ID: patientId -%} {% endif -%} - {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseID: patientId -%} + {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseId: patientId -%} {% if provenanceId -%} {% include 'Resource/Provenance' MSH: firstSegments.MSH, ORC: firstSegments.ORC, ID: provenanceId -%} {% endif -%} @@ -34,34 +34,39 @@ {% if encounterId -%} {% include 'Resource/Encounter' PV1: firstSegments.PV1, PV2: firstSegments.PV2, ID: encounterId -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.3 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId -%} + {% evaluate locationId3 using 'ID/Location' PL: firstSegments.PV1.3 -%} + {% if locationId3 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId3 -%} {% endif -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.6 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId -%} + {% evaluate locationId6 using 'ID/Location' PL: firstSegments.PV1.6 -%} + {% if locationId6 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId6 -%} {% endif -%} + {% include 'Resource/Encounter' PV1: firstSegments.PV1, Location_ID_PV1_3: locationId3, Location_ID_PV1_6: locationId6, ID: encounterId -%} + {% if patientId -%} {% include 'Reference/Encounter/Subject' ID: encounterId, REF: fullPatientId -%} {% endif -%} {% endif -%} - {% evaluate procedureId using 'ID/Procedure' PR1: firstSegments.PR1, baseId: patientId -%} - {% if procedureId -%} - {% include 'Resource/Procedure' PR1: firstSegments.PR1, ID: procedureId -%} + {% assign pr1SegmentLists = hl7v2Data | get_segment_lists: 'PR1' -%} + {% for pr1Segment in pr1SegmentLists.PR1 -%} + {% evaluate procedureId using 'ID/Procedure' PR1: pr1Segment, baseId: patientId -%} + {% if procedureId -%} + {% include 'Resource/Procedure' PR1: pr1Segment, ID: procedureId -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PR1.23 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PR1.23, ID: locationId -%} - {% endif -%} + {% evaluate locationId using 'ID/Location' PL: pr1Segment.23 -%} + {% if locationId -%} + {% include 'Resource/Location' PL: pr1Segment.23, ID: locationId -%} + {% endif -%} - {% if patientId -%} - {% include 'Reference/Procedure/Subject' ID: procedureId, REF: fullPatientId -%} + {% if patientId -%} + {% include 'Reference/Procedure/Subject' ID: procedureId, REF: fullPatientId -%} + {% endif -%} {% endif -%} - {% endif -%} + {% endfor -%} {% assign nk1SegmentLists = hl7v2Data | get_segment_lists: 'NK1' -%} {% for nk1Segment in nk1SegmentLists.NK1 -%} diff --git a/data/Templates/Hl7v2/OML_O21.liquid b/data/Templates/Hl7v2/OML_O21.liquid index 9f13fbf74..46ae7633c 100644 --- a/data/Templates/Hl7v2/OML_O21.liquid +++ b/data/Templates/Hl7v2/OML_O21.liquid @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ {% assign firstSegments = hl7v2Data | get_first_segments: 'PID|PD1|PV1|PV2|ORC|MSH' -%} @@ -15,7 +15,7 @@ {% include 'Resource/Patient' PID: firstSegments.PID, PD1: firstSegments.PD1, ID: patientId -%} {% endif -%} - {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseID: patientId -%} + {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseId: patientId -%} {% if provenanceId -%} {% include 'Resource/Provenance' MSH: firstSegments.MSH, ORC: firstSegments.ORC, ID: provenanceId -%} {% endif -%} @@ -35,16 +35,18 @@ {% assign fullEncounterId = encounterId | prepend: 'Encounter/' -%} {% include 'Resource/Encounter' PV1: firstSegments.PV1, PV2: firstSegments.PV2, ID: encounterId -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.3 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId -%} + {% evaluate locationId3 using 'ID/Location' PL: firstSegments.PV1.3 -%} + {% if locationId3 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId3 -%} {% endif -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.6 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId -%} + {% evaluate locationId6 using 'ID/Location' PL: firstSegments.PV1.6 -%} + {% if locationId6 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId6 -%} {% endif -%} + {% include 'Resource/Encounter' PV1: firstSegments.PV1, Location_ID_PV1_3: locationId3, Location_ID_PV1_6: locationId6, ID: encounterId -%} + {% if patientId -%} {% include 'Reference/Encounter/Subject' ID: encounterId, REF: fullPatientId -%} {% endif -%} diff --git a/data/Templates/Hl7v2/ORU_R01.liquid b/data/Templates/Hl7v2/ORU_R01.liquid index bbecd4ecc..ac366efba 100644 --- a/data/Templates/Hl7v2/ORU_R01.liquid +++ b/data/Templates/Hl7v2/ORU_R01.liquid @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ {% assign firstSegments = hl7v2Data | get_first_segments: 'PID|PD1|NK1|PV1|PV2|MSH' -%} @@ -15,7 +15,7 @@ {% include 'Resource/Patient' PID: firstSegments.PID, PD1: firstSegments.PD1, ID: patientId -%} {% endif -%} - {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseID: patientId -%} + {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseId: patientId -%} {% if provenanceId -%} {% include 'Resource/Provenance' MSH: firstSegments.MSH, ORC: firstSegments.ORC, ID: provenanceId -%} {% endif -%} @@ -35,16 +35,18 @@ {% assign fullEncounterId = encounterId | prepend: 'Encounter/' -%} {% include 'Resource/Encounter' PV1: firstSegments.PV1, PV2: firstSegments.PV2, ID: encounterId -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.3 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId -%} + {% evaluate locationId3 using 'ID/Location' PL: firstSegments.PV1.3 -%} + {% if locationId3 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId3 -%} {% endif -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.6 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId -%} + {% evaluate locationId6 using 'ID/Location' PL: firstSegments.PV1.6 -%} + {% if locationId6 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId6 -%} {% endif -%} - + + {% include 'Resource/Encounter' PV1: firstSegments.PV1, Location_ID_PV1_3: locationId3, Location_ID_PV1_6: locationId6, ID: encounterId -%} + {% if patientId -%} {% include 'Reference/Encounter/Subject' ID: encounterId, REF: fullPatientId -%} {% endif -%} diff --git a/data/Templates/Hl7v2/Resource/_Encounter.liquid b/data/Templates/Hl7v2/Resource/_Encounter.liquid index dde094796..819f7f0ce 100644 --- a/data/Templates/Hl7v2/Resource/_Encounter.liquid +++ b/data/Templates/Hl7v2/Resource/_Encounter.liquid @@ -18,25 +18,25 @@ { "location": { - {% if PV1.3 -%} - "reference":"Location/{{generateUUID PV1-3}}", + {% if Location_ID_PV1_3 -%} + "reference":"Location/{{ Location_ID_PV1_3 }}", {% endif -%} }, - {% if PV1.2.1.Value != "P" -%} + {% if PV1.2.1.Value != "P" and Location_ID_PV1_3 -%} "status":"active", {% endif -%} - {% if PV1.2.1.Value == "P" -%} + {% if PV1.2.1.Value == "P" and Location_ID_PV1_3 -%} "status":"planned", {% endif -%} }, { "location": { - {% if PV1.6 -%} - "reference":"Location/{{generateUUID PV1-6}}", + {% if Location_ID_PV1_6 -%} + "reference":"Location/{{ Location_ID_PV1_6 }}", {% endif -%} }, - {% if PV1.6 -%} + {% if PV1.6 and Location_ID_PV1_6 -%} "status":"completed", {% endif -%} }, diff --git a/data/Templates/Hl7v2/Resource/_Patient.liquid b/data/Templates/Hl7v2/Resource/_Patient.liquid index b54079a27..748748f21 100644 --- a/data/Templates/Hl7v2/Resource/_Patient.liquid +++ b/data/Templates/Hl7v2/Resource/_Patient.liquid @@ -42,9 +42,11 @@ ], "name": [ + {% for p in PID.5.Repeats -%} { - {% include 'DataType/XPN' XPN: PID.5 -%} + {% include 'DataType/XPN' XPN: p -%} }, + {% endfor -%} { {% include 'DataType/XPN' XPN: PID.9 -%} }, @@ -53,9 +55,11 @@ "gender":"{{ PID.8.Value | get_property: 'CodeSystem/Gender', 'code' }}", "address": [ + {% for p in PID.11.Repeats -%} { - {% include 'DataType/XAD' XAD: PID.11 -%} + {% include 'DataType/XAD' XAD: p -%} }, + {% endfor -%} { "district":"{{ PID.12.Value }}", }, diff --git a/data/Templates/Hl7v2/Resource/_PractitionerRole.liquid b/data/Templates/Hl7v2/Resource/_PractitionerRole.liquid index adecbe0cd..bcee04312 100644 --- a/data/Templates/Hl7v2/Resource/_PractitionerRole.liquid +++ b/data/Templates/Hl7v2/Resource/_PractitionerRole.liquid @@ -33,8 +33,8 @@ "location": [ { - {% if ROL.13 -%} - "reference":"Location/{{generateUUID ROL-13}}", + {% if Location_ID_ROL_13 -%} + "reference":"Location/{{ Location_ID_ROL_13 }}", {% endif -%} }, ], diff --git a/data/Templates/Hl7v2/Resource/_Procedure.liquid b/data/Templates/Hl7v2/Resource/_Procedure.liquid index 14e2ed5fa..8ea4f7c07 100644 --- a/data/Templates/Hl7v2/Resource/_Procedure.liquid +++ b/data/Templates/Hl7v2/Resource/_Procedure.liquid @@ -38,8 +38,8 @@ ], "location": { - {% if PR1.23 -%} - "reference":"Location/{{generateUUID PR1-23}}", + {% if Location_ID_PR1_23 -%} + "reference":"Location/{{ Location_ID_PR1_23 }}", {% endif -%} }, }, diff --git a/data/Templates/Hl7v2/VXU_V04.liquid b/data/Templates/Hl7v2/VXU_V04.liquid index 17a47419f..79780fd03 100644 --- a/data/Templates/Hl7v2/VXU_V04.liquid +++ b/data/Templates/Hl7v2/VXU_V04.liquid @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ {% assign firstSegments = hl7v2Data | get_first_segments: 'PID|PD1|PV1|ORC|MSH' -%} @@ -15,7 +15,7 @@ {% include 'Resource/Patient' PID: firstSegments.PID, PD1: firstSegments.PD1, ID: patientId -%} {% endif -%} - {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseID: patientId -%} + {% evaluate provenanceId using 'ID/Provenance' MSH: firstSegments.MSH, baseId: patientId -%} {% if provenanceId -%} {% include 'Resource/Provenance' MSH: firstSegments.MSH, ORC: firstSegments.ORC, ID: provenanceId -%} {% endif -%} @@ -34,15 +34,17 @@ {% if encounterId -%} {% include 'Resource/Encounter' PV1: firstSegments.PV1, ID: encounterId -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.3 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId -%} + {% evaluate locationId3 using 'ID/Location' PL: firstSegments.PV1.3 -%} + {% if locationId3 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.3, ID: locationId3 -%} {% endif -%} - {% evaluate locationId using 'ID/Location' PL: firstSegments.PV1.6 -%} - {% if locationId -%} - {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId -%} + {% evaluate locationId6 using 'ID/Location' PL: firstSegments.PV1.6 -%} + {% if locationId6 -%} + {% include 'Resource/Location' PL: firstSegments.PV1.6, ID: locationId6 -%} {% endif -%} + + {% include 'Resource/Encounter' PV1: firstSegments.PV1, Location_ID_PV1_3: locationId3, Location_ID_PV1_6: locationId6, ID: encounterId -%} {% if patientId -%} {% include 'Reference/Encounter/Subject' ID: encounterId, REF: fullPatientId -%} diff --git a/release.yml b/release.yml index 974a598a3..4d005992b 100644 --- a/release.yml +++ b/release.yml @@ -16,7 +16,7 @@ variables: functionalTests: "**/*FunctionalTests/*.csproj" buildConfiguration: 'Release' major: 3 - minor: 1 + minor: 2 bulidnum: $[counter(format('{0}.{1}',variables['major'],variables['minor']), 100)] revision: $[counter(format('{0:yyyyMMdd}', pipeline.startTime), 1)] version: $(major).$(minor).$(bulidnum).$(revision) @@ -29,6 +29,16 @@ stages: - job: Build steps: - script: echo $(version) + + - task: DotNetCoreCLI@2 + displayName: 'dotnet restore' + inputs: + command: 'restore' + projects: '$(solution)' + arguments: '--configuration $(buildConfiguration)' + feedsToUse: 'select' + vstsFeed: '7621b231-1a7d-4364-935b-2f72b911c43d/a60b7c8b-c6ae-4a8e-bd15-a526b603a1f2' + - task: DotNetCoreCLI@2 displayName: 'dotnet build' inputs: @@ -81,7 +91,7 @@ stages: includeRootFolder: false archiveType: 'tar' tarCompression: 'gz' - archiveFile: '$(Build.SourcesDirectory)/data/Templates/DefaultTemplates.tar.gz' + archiveFile: '$(Build.SourcesDirectory)/data/Templates/Hl7v2DefaultTemplates.tar.gz' - task: CopyFiles@2 displayName: 'copy DefaultTemplates to artifacts' @@ -127,7 +137,10 @@ stages: artifactName: FhirConverterBuild downloadPath: $(System.DefaultWorkingDirectory) - script: | - docker run --rm -d -p 5000:5000 --name registry stefanscherer/registry-windows:2.6.2 + git clone -q https://github.com/sowu880/dockerfiles-windows.git + cd dockerfiles-windows/registry/ + docker build -t registry-windows:2.7.1 . + docker run --rm -d -p 5000:5000 --name registry registry-windows:2.7.1 displayName: start registry # - script: | @@ -158,6 +171,11 @@ stages: artifactName: FhirConverterBuild downloadPath: $(System.DefaultWorkingDirectory) - script: | + curl -LO https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_linux_amd64.tar.gz + mkdir -p oras-install/ + tar -zxf oras_0.8.1_*.tar.gz -C oras-install/ + mv oras-install/oras /usr/local/bin/ + rm -rf oras_0.8.1_*.tar.gz oras-install/ docker run --rm -d -p 5000:5000 --name registry registry:2 displayName: start registry - script: | @@ -194,7 +212,7 @@ stages: tag: v$(major).$(minor).$(bulidnum) assets: | $(System.DefaultWorkingDirectory)/FhirConverterBuild/bin/** - $(System.DefaultWorkingDirectory)/FhirConverterBuild/data/Templates/DefaultTemplates.tar.gz + $(System.DefaultWorkingDirectory)/FhirConverterBuild/data/Templates/Hl7v2DefaultTemplates.tar.gz diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs index 1e21aa6ea..2ace659ff 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/FunctionalTests.cs @@ -86,5 +86,22 @@ public void GivenAnInvalidTemplate_WhenConverting_ExceptionsShouldBeThrown() var exception = Assert.Throws(() => hl7v2Processor.Convert(@"MSH|^~\&|", "template", new Hl7v2TemplateProvider(templateCollection))); Assert.True(exception.InnerException is DotLiquid.Exceptions.StackLevelException); } + + [Fact] + public void GivenEscapedMessage_WhenConverting_ExpectedCharacterShouldbeReturned() + { + var hl7v2Processor = new Hl7v2Processor(); + var templateDirectory = Path.Join(AppDomain.CurrentDomain.BaseDirectory, Constants.TemplateDirectory, "Hl7v2"); + var inputContent = string.Join("\n", new List + { + @"MSH|^~\&|FOO|BAR|FOO|BAR|20201225000000|FOO|ADT^A01|123456|P|2.3|||||||||||", + @"PR1|1|FOO|FOO^ESCAPED ONE \T\ ESCAPED TWO^BAR|ESCAPED THREE \T\ ESCAPED FOUR|20201225000000||||||||||", + }); + var result = JObject.Parse(hl7v2Processor.Convert(inputContent, "ADT_A01", new Hl7v2TemplateProvider(templateDirectory))); + + var texts = result.SelectTokens("$.entry[?(@.resource.resourceType == 'Procedure')].resource.code.text").Select(Convert.ToString); + var expected = new List { "ESCAPED ONE & ESCAPED TWO", "ESCAPED THREE & ESCAPED FOUR" }; + Assert.NotEmpty(texts.Intersect(expected)); + } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-23-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-23-expected.json index ef298d331..f92f29229 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-23-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-23-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:d3171c25-5db7-e705-140c-19d0972f1ba2", @@ -132,10 +132,10 @@ } }, { - "fullUrl": "urn:uuid:8530bde1-9b03-3292-264e-9b305cf30b37", + "fullUrl": "urn:uuid:b5aae309-4caf-c2cd-b087-0fd6db6dd058", "resource": { "resourceType": "Provenance", - "id": "8530bde1-9b03-3292-264e-9b305cf30b37", + "id": "b5aae309-4caf-c2cd-b087-0fd6db6dd058", "agent": [ { "type": { @@ -152,7 +152,7 @@ }, "request": { "method": "PUT", - "url": "Provenance/8530bde1-9b03-3292-264e-9b305cf30b37" + "url": "Provenance/b5aae309-4caf-c2cd-b087-0fd6db6dd058" } }, { @@ -193,7 +193,7 @@ "location": [ { "location": { - "reference": "Location/" + "reference": "Location/9b2956e2-230b-9dd5-ad8d-0e697a8664c1" }, "status": "active" } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-28-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-28-expected.json index 25b74f733..a50238d21 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-28-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT01-28-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:64c2d51b-34a4-e841-fb1c-03f1f7762d1a", @@ -157,10 +157,10 @@ } }, { - "fullUrl": "urn:uuid:8b872c00-30c9-392b-a5ca-ba6176cd7f75", + "fullUrl": "urn:uuid:a6091df2-6411-bec2-7f42-36d851d96cbe", "resource": { "resourceType": "Provenance", - "id": "8b872c00-30c9-392b-a5ca-ba6176cd7f75", + "id": "a6091df2-6411-bec2-7f42-36d851d96cbe", "agent": [ { "type": { @@ -177,7 +177,7 @@ }, "request": { "method": "PUT", - "url": "Provenance/8b872c00-30c9-392b-a5ca-ba6176cd7f75" + "url": "Provenance/a6091df2-6411-bec2-7f42-36d851d96cbe" } }, { @@ -239,7 +239,7 @@ "location": [ { "location": { - "reference": "Location/" + "reference": "Location/29eb5214-21d0-ed82-39f9-da4e50277c80" }, "status": "active" } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-23-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-23-expected.json index ede18c261..8c4085f52 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-23-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-23-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:18548d54-14a8-815a-c1bb-396f35b37164", @@ -162,10 +162,10 @@ } }, { - "fullUrl": "urn:uuid:dd338e52-023d-44ac-5ede-c7f279ec6829", + "fullUrl": "urn:uuid:ab3d28cb-c909-4ba2-b02e-0a868895ef0b", "resource": { "resourceType": "Provenance", - "id": "dd338e52-023d-44ac-5ede-c7f279ec6829", + "id": "ab3d28cb-c909-4ba2-b02e-0a868895ef0b", "agent": [ { "type": { @@ -182,7 +182,7 @@ }, "request": { "method": "PUT", - "url": "Provenance/dd338e52-023d-44ac-5ede-c7f279ec6829" + "url": "Provenance/ab3d28cb-c909-4ba2-b02e-0a868895ef0b" } }, { @@ -236,13 +236,13 @@ "location": [ { "location": { - "reference": "Location/" + "reference": "Location/1061dd71-d0a8-e8d0-7468-5be2eb4db1f8" }, "status": "active" }, { "location": { - "reference": "Location/" + "reference": "Location/affd96b8-87bc-956b-0215-7db7a8c6a9b3" }, "status": "completed" } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-251-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-251-expected.json index 9595a9031..22124d105 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-251-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-251-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:7393bcca-6d2f-543d-7787-0753133a8496", @@ -132,11 +132,6 @@ "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode" }, "status": "unknown", - "location": [ - { - "status": "active" - } - ], "participant": [ { "type": [ diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-28-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-28-expected.json index 457d9f8f0..b252174e6 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-28-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ADT_A01/ADT04-28-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:aa9b0327-0d9f-945a-888c-07658da974df", @@ -122,10 +122,10 @@ } }, { - "fullUrl": "urn:uuid:27610aa3-9d40-71d4-90ad-3feaff4411c7", + "fullUrl": "urn:uuid:a81dc37f-d7f3-c46a-00b1-4ff08fa99061", "resource": { "resourceType": "Provenance", - "id": "27610aa3-9d40-71d4-90ad-3feaff4411c7", + "id": "a81dc37f-d7f3-c46a-00b1-4ff08fa99061", "agent": [ { "type": { @@ -142,14 +142,14 @@ }, "request": { "method": "PUT", - "url": "Provenance/27610aa3-9d40-71d4-90ad-3feaff4411c7" + "url": "Provenance/a81dc37f-d7f3-c46a-00b1-4ff08fa99061" } }, { - "fullUrl": "urn:uuid:9ec7ccd5-cb9a-0a30-eb47-1755ff8f9f7c", + "fullUrl": "urn:uuid:256cf56f-8ec4-5af5-cf3f-4f9a833482be", "resource": { "resourceType": "Account", - "id": "9ec7ccd5-cb9a-0a30-eb47-1755ff8f9f7c", + "id": "256cf56f-8ec4-5af5-cf3f-4f9a833482be", "identifier": [ { "value": "1234" @@ -158,7 +158,7 @@ }, "request": { "method": "PUT", - "url": "Account/9ec7ccd5-cb9a-0a30-eb47-1755ff8f9f7c" + "url": "Account/256cf56f-8ec4-5af5-cf3f-4f9a833482be" } }, { @@ -175,7 +175,7 @@ "location": [ { "location": { - "reference": "Location/" + "reference": "Location/a821376a-5d7b-7274-8d95-a3f7c7efaf1b" }, "status": "active" } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-1-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-1-expected.json index 13880bafe..d0f45c0f1 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-1-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-1-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:6eda7d2f-32f8-a6c7-5125-d59f65a25c97", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-2-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-2-expected.json index 7a2bab091..68113075e 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-2-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/OML_O21/MDHHS-OML-O21-2-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:f651f710-5269-afaa-f743-57b7a501b0c6", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-1-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-1-expected.json index a4db11a50..90f656bb3 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-1-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-1-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:1a1787ae-e880-227d-1d8e-6167130b9a66", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-2-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-2-expected.json index 01b2a8696..4b86debb9 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-2-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LAB-ORU-2-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:9a62ebb2-108d-3a93-9edf-18645610f536", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LRI_2.0-NG_CBC_Typ_Message-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LRI_2.0-NG_CBC_Typ_Message-expected.json index b7c6588a6..9473edc5d 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LRI_2.0-NG_CBC_Typ_Message-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/LRI_2.0-NG_CBC_Typ_Message-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:15cde54f-eadb-4bee-fe4c-0e8dbe4f4959", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/ORU-R01-RMGEAD-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/ORU-R01-RMGEAD-expected.json index af0e67125..095f7ea18 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/ORU-R01-RMGEAD-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/ORU_R01/ORU-R01-RMGEAD-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:59137ac2-b595-49c0-863e-df25b62bf2cd", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/IZ_1_1.1_Admin_Child_Max_Message-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/IZ_1_1.1_Admin_Child_Max_Message-expected.json index 0dbb48126..9223ebfaf 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/IZ_1_1.1_Admin_Child_Max_Message-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/IZ_1_1.1_Admin_Child_Max_Message-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:5cddb2d1-c441-ec21-1c0c-96605dd3a9c0", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/VXU-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/VXU-expected.json index ba20b5344..f54a07918 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/VXU-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Hl7v2/VXU_V04/VXU-expected.json @@ -1,6 +1,6 @@ { "resourceType": "Bundle", - "type": "transaction", + "type": "batch", "entry": [ { "fullUrl": "urn:uuid:aefd5bbc-563b-7d19-5096-4b5754c338c1", diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs index a2a341c6e..6fbe0c8a7 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs @@ -53,7 +53,7 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh var context = new Context( environments: new List(), outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), + registers: Hash.FromDictionary(new Dictionary() { { "file_system", templateProvider.GetTemplateFileSystem() } }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: 0, @@ -90,7 +90,7 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown() context = new Context( environments: new List(), outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), + registers: Hash.FromDictionary(new Dictionary() { { "file_system", templateProvider.GetTemplateFileSystem() } }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: 0, diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs index abfbedc3f..9e225dc40 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Models/Hl7v2TraceInfoTests.cs @@ -51,6 +51,11 @@ public void GivenHl7v2Data_WhenCreate_CorrectHl7v2TraceInfoShouldBeReturned() Assert.Equal(2, traceInfo.UnusedSegments.Count); Assert.Equal(27, traceInfo.UnusedSegments[1].Components.Count); + // Specially test MSH unused segments + Assert.Equal(9, traceInfo.UnusedSegments[0].Components[0].Start); + Assert.Equal("AccMgr", traceInfo.UnusedSegments[0].Components[0].Value); + Assert.Equal(15, traceInfo.UnusedSegments[0].Components[0].End); + // Valid Hl7v2Data after render var processor = new Hl7v2Processor(); var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs index c7b4ff2a2..4995e38ec 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2DataParser.cs @@ -1,4 +1,4 @@ -// ------------------------------------------------------------------------------------------------- +// ------------------------------------------------------------------------------------------------- // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- @@ -36,7 +36,7 @@ public Hl7v2Data Parse(string message) for (var i = 0; i < segments.Length; ++i) { var fields = ParseFields(segments[i], encodingCharacters, isHeaderSegment: i == 0); - var segment = new Hl7v2Segment(segments[i], fields); + var segment = new Hl7v2Segment(NormalizeText(segments[i], encodingCharacters), fields); result.Meta.Add(fields.First()?.Value ?? string.Empty); result.Data.Add(segment); } @@ -81,16 +81,27 @@ private List ParseFields(string dataString, Hl7v2EncodingCharacters { if (!string.IsNullOrEmpty(fieldValues[f])) { - var components = ParseComponents(fieldValues[f], encodingCharacters); - var field = new Hl7v2Field(fieldValues[f], components); + /** + * We have four circumstances here. + * 1. The templates using this field treat it as repeatable, and the field contains $RepetitionSeparator; + * 2. The templates using this field treat it as repeatable, and the field doesn't contain $RepetitionSeparator; + * 3. The templates using this field treat it as unrepeatable, and the field contains $RepetitionSeparator; + * 4. The templates using this field treat it as unrepeatable, and the field doesn't contain $RepetitionSeparator; + * + * For circumstance #1 and #2, it will be all ok because the $field.Repeats always contains at least one value; + * For circumstance #3, we just take the first element in the repetition as the whole field by default; + * For circumstance #4, there will also be all right to take the first element of the $Repeats as the field itself; + */ + var field = new Hl7v2Field(NormalizeText(fieldValues[f], encodingCharacters), new List()); var repetitions = fieldValues[f].Split(encodingCharacters.RepetitionSeparator); for (var r = 0; r < repetitions.Length; ++r) { var repetitionComponents = ParseComponents(repetitions[r], encodingCharacters); - var repetition = new Hl7v2Field(repetitions[r], repetitionComponents); + var repetition = new Hl7v2Field(NormalizeText(repetitions[r], encodingCharacters), repetitionComponents); field.Repeats.Add(repetition); } + field.Components = ((Hl7v2Field)field.Repeats[0]).Components; fields.Add(field); } else @@ -113,7 +124,7 @@ private List ParseComponents(string dataString, Hl7v2EncodingCha if (!string.IsNullOrEmpty(componentValue)) { var subcomponents = ParseSubcomponents(componentValue, encodingCharacters); - var component = new Hl7v2Component(componentValue, subcomponents); + var component = new Hl7v2Component(NormalizeText(componentValue, encodingCharacters), subcomponents); components.Add(component); } else @@ -132,9 +143,7 @@ private List ParseSubcomponents(string dataString, Hl7v2EncodingCharacte var subcomponentValues = dataString.Split(encodingCharacters.SubcomponentSeparator); foreach (var subcomponentValue in subcomponentValues) { - subcomponents.Add( - SpecialCharProcessor.Escape( - Hl7v2EscapeSequenceProcessor.Unescape(subcomponentValue, encodingCharacters))); + subcomponents.Add(NormalizeText(subcomponentValue, encodingCharacters)); } return subcomponents; @@ -151,5 +160,12 @@ private Hl7v2EncodingCharacters ParseHl7v2EncodingCharacters(string headerSegmen SubcomponentSeparator = headerSegment[7], }; } + + private string NormalizeText(string value, Hl7v2EncodingCharacters encodingCharacters) + { + var semanticalUnescape = Hl7v2EscapeSequenceProcessor.Unescape(value, encodingCharacters); + var grammarEscape = SpecialCharProcessor.Escape(semanticalUnescape); + return grammarEscape; + } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs index cf182d5eb..4258a725f 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs @@ -66,11 +66,11 @@ public string Convert(string data, string rootTemplate, ITemplateProvider templa private Context CreateContext(ITemplateProvider templateProvider, Hl7v2Data hl7v2Data) { // Load data and templates - var timeout = _settings != null ? _settings.TimeOut : 0; + var timeout = _settings?.TimeOut ?? 0; var context = new Context( - environments: new List() { Hash.FromAnonymousObject(new { hl7v2Data }) }, + environments: new List() { Hash.FromDictionary(new Dictionary() { { "hl7v2Data", hl7v2Data } }) }, outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), + registers: Hash.FromDictionary(new Dictionary() { { "file_system", templateProvider.GetTemplateFileSystem() } }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: timeout, diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs index 0f22f3b81..28cf65884 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Models/Hl7v2TraceInfo.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Extensions; +using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.InputProcessor; using Microsoft.Health.Fhir.Liquid.Converter.Models; namespace Microsoft.Health.Fhir.Liquid.Converter.Hl7v2.Models @@ -31,20 +32,16 @@ public static Hl7v2TraceInfo CreateTraceInfo(Hl7v2Data hl7v2Data) { for (var i = 0; i < hl7v2Data?.Data?.Count; ++i) { + var segmentHeader = hl7v2Data.Meta[i]; var segment = hl7v2Data.Data[i]; var unusedSegment = new UnusedHl7v2Segment(i); + unusedSegment.Type = segmentHeader; + for (var j = 0; j < segment?.Fields?.Count; ++j) { - // Encoding characters field is treated as accessed - if (i == 0 && j == 1) - { - continue; - } - - // Segment id field is treated as accessed - if (j == 0 && segment.Fields[j] is Hl7v2Field segmentIdField) + // Field separator and encoding characters field are treated as accessed + if (i == 0 && j <= 2) { - unusedSegment.Type = segmentIdField.Value; continue; } @@ -55,7 +52,7 @@ public static Hl7v2TraceInfo CreateTraceInfo(Hl7v2Data hl7v2Data) { if (field.Components[k] is Hl7v2Component component && component.IsAccessed == false) { - var indexInSegment = FindOffsetInSegment(segment.Value, hl7v2Data.EncodingCharacters, j, k - 1); + var indexInSegment = FindOffsetInSegment(segmentHeader, segment.Value, hl7v2Data.EncodingCharacters, j, k - 1); var unusedComponent = new UnusedHl7v2Component(indexInSegment, indexInSegment + component.Value.Length, component.Value); unusedComponents.Add(unusedComponent); } @@ -82,8 +79,18 @@ public static Hl7v2TraceInfo CreateTraceInfo(Hl7v2Data hl7v2Data) return new Hl7v2TraceInfo(unusedSegments); } - private static int FindOffsetInSegment(string segmentValue, Hl7v2EncodingCharacters encodingCharacters, int fieldIndex, int componentIndex) + private static int FindOffsetInSegment(string segmentHeader, string segmentValue, Hl7v2EncodingCharacters encodingCharacters, int fieldIndex, int componentIndex) { + // All values($segmentValue and $fieldValue) need to be unescaped firstly (from "\\" back to "\"), + // or the length will be incorrectly calculated + segmentValue = SpecialCharProcessor.Unescape(segmentValue); + + // MSH segment should be treated separately since the first '|' in MSH segment is a special field + if (segmentHeader.Equals("MSH")) + { + fieldIndex--; + } + var startFieldIndex = segmentValue.IndexOfNthOccurrence(encodingCharacters.FieldSeparator, fieldIndex) + 1; var endFieldIndex = segmentValue.IndexOfNthOccurrence(encodingCharacters.FieldSeparator, fieldIndex + 1); if (endFieldIndex == -1) @@ -91,7 +98,7 @@ private static int FindOffsetInSegment(string segmentValue, Hl7v2EncodingCharact endFieldIndex = segmentValue.Length; } - var fieldValue = segmentValue.Substring(startFieldIndex, endFieldIndex - startFieldIndex); + var fieldValue = SpecialCharProcessor.Unescape(segmentValue.Substring(startFieldIndex, endFieldIndex - startFieldIndex)); return startFieldIndex + fieldValue.IndexOfNthOccurrence(encodingCharacters.ComponentSeparator, componentIndex) + 1; } } diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/ContainerRegistry.cs b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/ContainerRegistry.cs index 955f67e45..cd8a974ca 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/ContainerRegistry.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/ContainerRegistry.cs @@ -37,14 +37,8 @@ public ContainerRegistryInfo GetTestContainerRegistryInfo() return containerRegistry; } - public async Task GenerateTemplateImageAsync(string imageReference, List templateFiles) + public async Task GenerateTemplateImageAsync(ContainerRegistryInfo registry, string imageReference, List templateFiles) { - var registry = GetTestContainerRegistryInfo(); - if (registry == null) - { - return; - } - ImageInfo imageInfo = ImageInfo.CreateFromImageReference(imageReference); if (imageInfo.Registry != registry.ContainerRegistryServer) @@ -137,4 +131,4 @@ public override Task ProcessHttpRequestAsync(HttpRequestMessage request, Cancell } } } -} +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj index f44054780..92b4fb637 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj +++ b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests.csproj @@ -6,6 +6,12 @@ false + + ..\..\bin\Microsoft.Health.Fhir.TemplateManagement.FunctionalTests\ + false + false + + diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/OCIArtifactFunctionalTests.cs b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/OCIArtifactFunctionalTests.cs new file mode 100644 index 000000000..4012c41a1 --- /dev/null +++ b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/OCIArtifactFunctionalTests.cs @@ -0,0 +1,344 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.TemplateManagement.Client; +using Microsoft.Health.Fhir.TemplateManagement.Exceptions; +using Microsoft.Health.Fhir.TemplateManagement.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.TemplateManagement.FunctionalTests +{ + public class OCIArtifactFunctionalTests : IAsyncLifetime + { + private readonly string _containerRegistryServer; + private readonly string _baseLayerTemplatePath = "TestData/TarGzFiles/layer1.tar.gz"; + private readonly string _userLayerTemplatePath = "TestData/TarGzFiles/layer2.tar.gz"; + private readonly string _emptySequenceNumberLayerPath = "TestData/TarGzFiles/userV1.tar.gz"; + private readonly string _invalidCompressedImageLayerPath = "TestData/TarGzFiles/invalid1.tar.gz"; + private readonly string _testOneLayerWithValidSequenceNumberImageReference; + private readonly string _testOneLayerWithoutSequenceNumberImageReference; + private readonly string _testOneLayerWithInValidSequenceNumberImageReference; + private readonly string _testMultiLayersWithValidSequenceNumberImageReference; + private readonly string _testMultiLayersWithInValidSequenceNumberImageReference; + private readonly string _testInvalidCompressedImageReference; + private bool _isOrasValid = true; + private readonly string _orasErrorMessage = "Oras tool invalid."; + + public OCIArtifactFunctionalTests() + { + _containerRegistryServer = "localhost:5000"; + _testOneLayerWithValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:onelayer_valid_sequence"; + _testOneLayerWithoutSequenceNumberImageReference = _containerRegistryServer + "/templatetest:onelayer_without_sequence"; + _testOneLayerWithInValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:onelayer_invalid_sequence"; + _testMultiLayersWithValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:multilayers_valid_sequence"; + _testMultiLayersWithInValidSequenceNumberImageReference = _containerRegistryServer + "/templatetest:multilayers_invalid_sequence"; + _testInvalidCompressedImageReference = _containerRegistryServer + "/templatetest:invalid_image"; + } + + public async Task InitializeAsync() + { + await PushOneLayerWithValidSequenceNumberAsync(); + await PushOneLayerWithoutSequenceNumberAsync(); + await PushOneLayerWithInvalidSequenceNumberAsync(); + await PushMultiLayersWithValidSequenceNumberAsync(); + await PushMultiLayersWithInValidSequenceNumberAsync(); + await PushInvalidCompressedImageAsync(); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + private async Task PushOneLayerWithValidSequenceNumberAsync() + { + string command = $"push {_testOneLayerWithValidSequenceNumberImageReference} {_baseLayerTemplatePath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task PushOneLayerWithoutSequenceNumberAsync() + { + string command = $"push {_testOneLayerWithoutSequenceNumberImageReference} {_emptySequenceNumberLayerPath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task PushOneLayerWithInvalidSequenceNumberAsync() + { + string command = $"push {_testOneLayerWithInValidSequenceNumberImageReference} {_userLayerTemplatePath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task PushMultiLayersWithValidSequenceNumberAsync() + { + string command = $"push {_testMultiLayersWithValidSequenceNumberImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task PushMultiLayersWithInValidSequenceNumberAsync() + { + string command = $"push {_testMultiLayersWithInValidSequenceNumberImageReference} {_emptySequenceNumberLayerPath} {_userLayerTemplatePath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task PushInvalidCompressedImageAsync() + { + string command = $"push {_testInvalidCompressedImageReference} {_invalidCompressedImageLayerPath}"; + await ExecuteOrasCommandAsync(command); + } + + private async Task ExecuteOrasCommandAsync(string command) + { + try + { + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + } + catch + { + _isOrasValid = false; + } + } + + // Pull one layer image with valid sequence number, successfully pulled with base layer copied. + [Fact] + public async Task GivenOneLayerImageWithValidSequenceNumber_WhenPulled_ArtifactsWillBePulledWithBaseLayerCopiedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testOneLayerWithValidSequenceNumberImageReference; + string outputFolder = "TestData/testOneLayerWithValidSequenceNumber"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + Assert.Equal(842, Directory.EnumerateFiles(outputFolder, "*.*", SearchOption.AllDirectories).Count()); + Assert.Single(Directory.EnumerateFiles(Path.Combine(outputFolder, ".image", "base"), "*.tar.gz", SearchOption.AllDirectories)); + ClearFolder(outputFolder); + } + + // Pull one layer image without sequence number, successfully pulled without base layer copied. + [Fact] + public async Task GivenOneLayerImageWithoutSequenceNumber_WhenPulled_ArtifactsWillBePulledWithoutBaseLayerCopiedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testOneLayerWithoutSequenceNumberImageReference; + string outputFolder = "TestData/testOneLayerWithoutSequenceNumber"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + Assert.Equal(2, Directory.EnumerateFiles(outputFolder, "*.*", SearchOption.AllDirectories).Count()); + Assert.False(Directory.Exists(Path.Combine(outputFolder, ".image", "base"))); + ClearFolder(outputFolder); + } + + // Pull one layer image with invalid sequence number, exception will be thrown. + [Fact] + public async Task GivenOneLayerImageWithInvalidSequenceNumber_WhenPulled_ExceptionWillBeThrownAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testOneLayerWithInValidSequenceNumberImageReference; + string outputFolder = "TestData/testOneLayerWithInValidSequenceNumber"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + Assert.Throws(() => testManager.UnpackOCIImage()); + ClearFolder(outputFolder); + } + + // Pull multi-layers with valid sequence numbers, successfully pulled with base layer copied. + [Fact] + public async Task GivenMultiLayerImageWithValidSequenceNumber_WhenPulled_ArtifactsWillBePulledWithBaseLayerCopiedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testMultiLayersWithValidSequenceNumberImageReference; + string outputFolder = "TestData/testMultiLayersWithValidSequenceNumber"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + Assert.Equal(9, Directory.EnumerateFiles(outputFolder, "*.*", SearchOption.AllDirectories).Count()); + Assert.Single(Directory.EnumerateFiles(Path.Combine(outputFolder, ".image", "base"), "*.tar.gz", SearchOption.AllDirectories)); + ClearFolder(outputFolder); + } + + // Pull multi-layers with invalid sequence numbers, exception will be thrown. + [Fact] + public async Task GivenMultiLayersImageWithInvalidSequenceNumber_WhenPulled_ExceptionWillBeThrownAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testMultiLayersWithInValidSequenceNumberImageReference; + string outputFolder = "TestData/testMultiLayersWithInValidSequenceNumber"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + Assert.Throws(() => testManager.UnpackOCIImage()); + ClearFolder(outputFolder); + } + + // Pull invalid image, exception will be thrown. + [Fact] + public async Task GivenInvalidCompressedImage_WhenPulled_ExceptionWillBeThrownAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string imageReference = _testInvalidCompressedImageReference; + string outputFolder = "TestData/testInvalidCompressedImage"; + var testManager = new OCIFileManager(imageReference, outputFolder); + await testManager.PullOCIImageAsync(); + Assert.Throws(() => testManager.UnpackOCIImage()); + ClearFolder(outputFolder); + } + + // Push artifacts which unpacked from base layer. If user modify artifacts, successfully pushed multi-layers image. + [Fact] + public async Task GivenAnInputFolderUnpackedFromBaseLayer_WhenPushOCIFiles_IfUserModify_TwoLayersWillBePushedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + + // Pull an image + string initImageReference = _testOneLayerWithValidSequenceNumberImageReference; + string initInputFolder = "TestData/UserFolder1"; + var testManager = new OCIFileManager(initImageReference, initInputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + + // Modified by user + File.WriteAllText(Path.Combine(initInputFolder, "add"), "add"); + File.WriteAllText(Path.Combine(initInputFolder, "metadata.json"), "modify"); + File.Delete(Path.Combine(initInputFolder, "ADT_A01.liquid")); + + // Push new image. + string testPushMultiLayersImageReference = _containerRegistryServer + "/templatetest:push_multilayers"; + var pushManager = new OCIFileManager(testPushMultiLayersImageReference, initInputFolder); + pushManager.PackOCIImage(); + await pushManager.PushOCIImageAsync(); + + // Check Image + string command = $"pull {testPushMultiLayersImageReference} -o checkMultiLayersFolder"; + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + Assert.Equal(2, Directory.EnumerateFiles("checkMultiLayersFolder", "*.tar.gz", SearchOption.AllDirectories).Count()); + Assert.Equal(4, StreamUtility.DecompressTarGzStream(File.OpenRead(Path.Combine("checkMultiLayersFolder", "layer2.tar.gz"))).Count()); + ClearFolder(initInputFolder); + ClearFolder("checkMultiLayersFolder"); + } + + // Push artifacts and ignore base layer, successfully pushed one-layer image. + [Fact] + public async Task GivenAnInputFolderUnpackedFromBaseLayer_WhenPushOCIFiles_IfIgnoreBaseLayer_NewBaseLayerWillBePushedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + + // Pull an image + string initImageReference = _testOneLayerWithValidSequenceNumberImageReference; + string initInputFolder = "TestData/UserFolder2"; + var testManager = new OCIFileManager(initImageReference, initInputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + + // Modified by user + File.WriteAllText(Path.Combine(initInputFolder, "add"), "add"); + File.WriteAllText(Path.Combine(initInputFolder, "metadata.json"), "modify"); + File.Delete(Path.Combine(initInputFolder, "ADT_A01.liquid")); + + // Push new image ignore base layer. + string testPushNewBaseLayerImageReference = _containerRegistryServer + "/templatetest:push_newbaselayer"; + var pushManager = new OCIFileManager(testPushNewBaseLayerImageReference, initInputFolder); + pushManager.PackOCIImage(true); + await pushManager.PushOCIImageAsync(); + + // Check Image + string command = $"pull {testPushNewBaseLayerImageReference} -o checkNewBaseLayerFolder"; + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + Assert.Single(Directory.EnumerateFiles("checkNewBaseLayerFolder", "*.tar.gz", SearchOption.AllDirectories)); + Assert.Equal(840, StreamUtility.DecompressTarGzStream(File.OpenRead(Path.Combine("checkNewBaseLayerFolder", "layer1.tar.gz"))).Count()); + ClearFolder(initInputFolder); + ClearFolder("checkNewBaseLayerFolder"); + } + + // Push artifacts which unpacked from base layer. If user don't modify artifacts, successfully pushed one-layer image. + [Fact] + public async Task GivenAnInputFolderUnpackedFromBaseLayer_WhenPushOCIFiles_IfUserDoNotModify_OnlyBaseLayerWillBePushedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + + // Pull an image + string initImageReference = _testOneLayerWithValidSequenceNumberImageReference; + string initInputFolder = "TestData/UserFolder3"; + var testManager = new OCIFileManager(initImageReference, initInputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + + // Push image. + string testPushBaseLayerImageReference = _containerRegistryServer + "/templatetest:push_baselayer"; + var pushManager = new OCIFileManager(testPushBaseLayerImageReference, initInputFolder); + pushManager.PackOCIImage(); + await pushManager.PushOCIImageAsync(); + + // Check Image + string command = $"pull {testPushBaseLayerImageReference} -o checkBaseLayerFolder"; + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + Assert.Single(Directory.EnumerateFiles("checkBaseLayerFolder", "*.tar.gz", SearchOption.AllDirectories)); + ClearFolder(initInputFolder); + ClearFolder("checkBaseLayerFolder"); + } + + // Push artifacts without base layer, successfully pushed one-layer image. + [Fact] + public async Task GivenAnInputFolderWithoutBaseLayer_WhenPushOCIFiles_IfUserModify_OneBaseLayerWillBePushedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + + // Pull an image + string initImageReference = _testOneLayerWithoutSequenceNumberImageReference; + string initInputFolder = "TestData/UserFolder4"; + var testManager = new OCIFileManager(initImageReference, initInputFolder); + await testManager.PullOCIImageAsync(); + testManager.UnpackOCIImage(); + + // Modified by user + File.WriteAllText(Path.Combine(initInputFolder, "add"), "add"); + + // Push image. + string testPushNewBaseLayerImageReference = _containerRegistryServer + "/templatetest:pushwithoutbase_newbaselayer"; + var pushManager = new OCIFileManager(testPushNewBaseLayerImageReference, initInputFolder); + pushManager.PackOCIImage(); + await pushManager.PushOCIImageAsync(); + + // Check Image + string command = $"pull {testPushNewBaseLayerImageReference} -o checkLayerFolder"; + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + Assert.Single(Directory.EnumerateFiles("checkLayerFolder", "*.tar.gz", SearchOption.AllDirectories)); + ClearFolder(initInputFolder); + ClearFolder("checkLayerFolder"); + } + + // Push empty artifact folder, exception will be thrown. + [Fact] + public async Task GivenAnEmptyInputFolder_WhenPushOCIFiles_OneBaseLayerWillBePushedAsync() + { + Assert.True(_isOrasValid, _orasErrorMessage); + string emptyFolder = "emptyFoler"; + Directory.CreateDirectory(emptyFolder); + + // Push image. + string testPushNewBaseLayerImageReference = _containerRegistryServer + "/templatetest:empty"; + var pushManager = new OCIFileManager(testPushNewBaseLayerImageReference, emptyFolder); + pushManager.PackOCIImage(); + await Assert.ThrowsAsync(() => pushManager.PushOCIImageAsync()); + + ClearFolder(emptyFolder); + } + + private void ClearFolder(string directory) + { + EnsureArg.IsNotNullOrEmpty(directory, nameof(directory)); + + if (!Directory.Exists(directory)) + { + return; + } + + DirectoryInfo folder = new DirectoryInfo(directory); + folder.Delete(true); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/FunctionalTests.cs b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TemplateCollectionFunctionalTests.cs similarity index 77% rename from src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/FunctionalTests.cs rename to src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TemplateCollectionFunctionalTests.cs index b35a0b8e1..6dae69bbd 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/FunctionalTests.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TemplateCollectionFunctionalTests.cs @@ -8,6 +8,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Threading; using System.Threading.Tasks; using DotLiquid; using Microsoft.Extensions.Caching.Memory; @@ -20,7 +21,7 @@ namespace Microsoft.Health.Fhir.TemplateManagement.FunctionalTests { - public class FunctionalTests + public class TemplateCollectionFunctionalTests : IAsyncLifetime { private readonly string token; private readonly TemplateCollectionConfiguration _config = new TemplateCollectionConfiguration(); @@ -33,12 +34,13 @@ public class FunctionalTests private readonly string testMultiLayerImageReference; private readonly string testInvalidImageReference; private readonly string testInvalidTemplateImageReference; - private readonly ContainerRegistry _containerRegistry; + private readonly ContainerRegistry _containerRegistry = new ContainerRegistry(); private readonly ContainerRegistryInfo _containerRegistryInfo; + private static readonly string _templateDirectory = Path.Join("..", "..", "data", "Templates"); + private static readonly string _sampleDataDirectory = Path.Join("..", "..", "data", "SampleData"); - public FunctionalTests() + public TemplateCollectionFunctionalTests() { - _containerRegistry = new ContainerRegistry(); _containerRegistryInfo = _containerRegistry.GetTestContainerRegistryInfo(); if (_containerRegistryInfo == null) { @@ -50,34 +52,24 @@ public FunctionalTests() testInvalidImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:invalidlayers"; testInvalidTemplateImageReference = _containerRegistryInfo.ContainerRegistryServer + "/templatetest:invalidtemplateslayers"; token = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_containerRegistryInfo.ContainerRegistryUsername}:{_containerRegistryInfo.ContainerRegistryPassword}")); - Task.Run(InitOneLayerImageAsync).Wait(); - Task.Run(InitMultiLayerImageAsync).Wait(); - Task.Run(InitInvalidTarGzImageAsync).Wait(); - Task.Run(InitInvalidTemplateImageAsync).Wait(); } - private async Task InitOneLayerImageAsync() + public async Task InitializeAsync() { - List templateFiles = new List { baseLayerTemplatePath }; - await _containerRegistry.GenerateTemplateImageAsync(testOneLayerImageReference, templateFiles); - } + if (_containerRegistryInfo == null) + { + return; + } - private async Task InitMultiLayerImageAsync() - { - List templateFiles = new List { baseLayerTemplatePath, userLayerTemplatePath }; - await _containerRegistry.GenerateTemplateImageAsync(testMultiLayerImageReference, templateFiles); + await InitOneLayerImageAsync(); + await InitMultiLayerImageAsync(); + await InitInvalidTarGzImageAsync(); + await InitInvalidTemplateImageAsync(); } - private async Task InitInvalidTarGzImageAsync() + public Task DisposeAsync() { - List templateFiles = new List { invalidTarGzPath }; - await _containerRegistry.GenerateTemplateImageAsync(testInvalidImageReference, templateFiles); - } - - private async Task InitInvalidTemplateImageAsync() - { - List templateFiles = new List { invalidTemplatePath }; - await _containerRegistry.GenerateTemplateImageAsync(testInvalidTemplateImageReference, templateFiles); + return Task.CompletedTask; } public static IEnumerable GetValidImageInfoWithTag() @@ -86,20 +78,37 @@ public static IEnumerable GetValidImageInfoWithTag() yield return new object[] { new List { 767, 838 }, "templatetest", "multilayers" }; } - public static IEnumerable GetHl7v2DataAndTemplateImageReference() + public static IEnumerable GetHl7v2DataAndEntryTemplate() { - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\ADT01-23.hl7", "ADT_A01" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\IZ_1_1.1_Admin_Child_Max_Message.hl7", "VXU_V04" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\LAB-ORU-1.hl7", "ORU_R01" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\MDHHS-OML-O21-1.hl7", "OML_O21" }; + var data = new List + { + new object[] { @"ADT01-23.hl7", @"ADT_A01" }, + new object[] { @"IZ_1_1.1_Admin_Child_Max_Message.hl7", @"VXU_V04" }, + new object[] { @"LAB-ORU-1.hl7", @"ORU_R01" }, + new object[] { @"MDHHS-OML-O21-1.hl7", @"OML_O21" }, + }; + return data.Select(item => new object[] + { + Path.Join(_sampleDataDirectory, "Hl7v2", Convert.ToString(item[0])), + Convert.ToString(item[1]), + }); } - public static IEnumerable GetHl7v2DataAndTemplateImageReferenceWithoutGivenTemplate() + public static IEnumerable GetHl7v2DataAndTemplateSources() { - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\ADT01-23.hl7", "ADT_A01" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\IZ_1_1.1_Admin_Child_Max_Message.hl7", "VXU_V04" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\LAB-ORU-1.hl7", "ORU_R01" }; - yield return new object[] { @"..\..\..\..\..\data\SampleData\Hl7v2\MDHHS-OML-O21-1.hl7", "OML_O21" }; + var data = new List + { + new object[] { @"ADT01-23.hl7", @"ADT_A01" }, + new object[] { @"IZ_1_1.1_Admin_Child_Max_Message.hl7", @"VXU_V04" }, + new object[] { @"LAB-ORU-1.hl7", @"ORU_R01" }, + new object[] { @"MDHHS-OML-O21-1.hl7", @"OML_O21" }, + }; + return data.Select(item => new object[] + { + Path.Join(_sampleDataDirectory, "Hl7v2", Convert.ToString(item[0])), + Path.Join(_templateDirectory, "Hl7v2"), + Convert.ToString(item[1]), + }); } public static IEnumerable GetNotExistImageInfo() @@ -230,7 +239,7 @@ public async Task GiveImageReference_WhenGetTemplateCollection_ACorrectTemplateC } [Theory] - [MemberData(nameof(GetHl7v2DataAndTemplateImageReference))] + [MemberData(nameof(GetHl7v2DataAndEntryTemplate))] public async Task GetTemplateCollectionFromACR_WhenGivenHl7v2DataForConverting__ExpectedFhirResourceShouldBeReturnedAsync(string hl7v2Data, string entryTemplate) { if (_containerRegistryInfo == null) @@ -245,7 +254,7 @@ public async Task GetTemplateCollectionFromACR_WhenGivenHl7v2DataForConverting__ } [Theory] - [MemberData(nameof(GetHl7v2DataAndTemplateImageReferenceWithoutGivenTemplate))] + [MemberData(nameof(GetHl7v2DataAndEntryTemplate))] public async Task GetTemplateCollectionFromACR_WhenGivenHl7v2DataForConverting_IfTemplateNotExist_ExceptionWillBeThrownAsync(string hl7v2Data, string entryTemplate) { if (_containerRegistryInfo == null) @@ -275,6 +284,25 @@ public async Task GiveDefaultImageReference_WhenGetTemplateCollectionWithEmptyTo Assert.Equal(defaultTemplatesCounts, templateCollection.First().Count()); } + // Conversion results of DefaultTemplates.tar.gz and default template folder should be the same. + [Theory] + [MemberData(nameof(GetHl7v2DataAndTemplateSources))] + public async Task GivenSameInputData_WithDifferentTemplateSource_WhenConvert_ResultShouldBeIdentical(string inputFile, string defaultTemplateDirectory, string rootTemplate) + { + var folderTemplateProvider = new Hl7v2TemplateProvider(defaultTemplateDirectory); + + var templateProviderFactory = new TemplateCollectionProviderFactory(new MemoryCache(new MemoryCacheOptions()), Options.Create(new TemplateCollectionConfiguration())); + var templateProvider = templateProviderFactory.CreateTemplateCollectionProvider(ImageInfo.DefaultTemplateImageReference, string.Empty); + var imageTemplateProvider = new Hl7v2TemplateProvider(await templateProvider.GetTemplateCollectionAsync(CancellationToken.None)); + + var hl7v2Processor = new Hl7v2Processor(); + var inputContent = File.ReadAllText(inputFile); + + var imageResult = hl7v2Processor.Convert(inputContent, rootTemplate, imageTemplateProvider); + var folderResult = hl7v2Processor.Convert(inputContent, rootTemplate, folderTemplateProvider); + Assert.Equal(imageResult, folderResult); + } + private void TestByTemplate(string inputFile, string entryTemplate, List> templateProvider) { var hl7v2Processor = new Hl7v2Processor(); @@ -283,5 +311,29 @@ private void TestByTemplate(string inputFile, string entryTemplate, List templateFiles = new List { baseLayerTemplatePath }; + await _containerRegistry.GenerateTemplateImageAsync(_containerRegistryInfo, testOneLayerImageReference, templateFiles); + } + + private async Task InitMultiLayerImageAsync() + { + List templateFiles = new List { baseLayerTemplatePath, userLayerTemplatePath }; + await _containerRegistry.GenerateTemplateImageAsync(_containerRegistryInfo, testMultiLayerImageReference, templateFiles); + } + + private async Task InitInvalidTarGzImageAsync() + { + List templateFiles = new List { invalidTarGzPath }; + await _containerRegistry.GenerateTemplateImageAsync(_containerRegistryInfo, testInvalidImageReference, templateFiles); + } + + private async Task InitInvalidTemplateImageAsync() + { + List templateFiles = new List { invalidTemplatePath }; + await _containerRegistry.GenerateTemplateImageAsync(_containerRegistryInfo, testInvalidTemplateImageReference, templateFiles); + } } } \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer1.tar.gz b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer1.tar.gz new file mode 100644 index 000000000..eb8d3f44f Binary files /dev/null and b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer1.tar.gz differ diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer2.tar.gz b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer2.tar.gz new file mode 100644 index 000000000..fe2a621e7 Binary files /dev/null and b/src/Microsoft.Health.Fhir.TemplateManagement.FunctionalTests/TestData/TarGzFiles/layer2.tar.gz differ diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/Client/OrasClientTests.cs b/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/Client/OrasClientTests.cs index 46a159e75..65f931025 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/Client/OrasClientTests.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/Client/OrasClientTests.cs @@ -5,9 +5,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Linq; using System.Threading.Tasks; using Microsoft.Health.Fhir.TemplateManagement.Client; using Microsoft.Health.Fhir.TemplateManagement.Exceptions; @@ -15,7 +13,7 @@ namespace Microsoft.Health.Fhir.TemplateManagement.UnitTests.Client { - public class OrasClientTests + public class OrasClientTests : IAsyncLifetime { private readonly string _containerRegistryServer; private readonly string _baseLayerTemplatePath = "TestData/TarGzFiles/baseLayer.tar.gz"; @@ -29,8 +27,17 @@ public OrasClientTests() _containerRegistryServer = Environment.GetEnvironmentVariable("TestContainerRegistryServer"); _testOneLayerImageReference = _containerRegistryServer + "/templatetest:v1"; _testMultiLayersImageReference = _containerRegistryServer + "/templatetest:v2"; - PushOneLayerImage(); - PushMultiLayersImage(); + } + + public async Task InitializeAsync() + { + await PushOneLayerImageAsync(); + await PushMultiLayersImageAsync(); + } + + public Task DisposeAsync() + { + return Task.CompletedTask; } public static IEnumerable GetInvalidReference() @@ -55,18 +62,6 @@ public static IEnumerable GetInValidFolder() yield return new object[] { @" " }; } - private void PushOneLayerImage() - { - string command = $"push {_testOneLayerImageReference} {_baseLayerTemplatePath}"; - OrasExecution(command); - } - - private void PushMultiLayersImage() - { - string command = $"push {_testMultiLayersImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}"; - OrasExecution(command); - } - [Theory] [MemberData(nameof(GetInValidFolder))] public async Task GivenInValidOutputFolder_WhenPullUseOras_ExceptionWillBeThrownAsync(string outputFolder) @@ -90,6 +85,7 @@ public async Task GivenValidOutputFolder_WhenPullImageUseOras_ImageWillBePulledA return; } + outputFolder = "testpull" + outputFolder; string imageReference = _testOneLayerImageReference; OrasClient orasClient = new OrasClient(imageReference); var ex = await Record.ExceptionAsync(async () => await orasClient.PullImageAsync(outputFolder)); @@ -126,6 +122,7 @@ public async Task GivenValidInputFolder_WhenPushUseOras_ImageWillBePushedAsync(s return; } + outputFolder = "testpush" + outputFolder; string imageReference = _testOneLayerImageReference; OrasClient orasClient = new OrasClient(imageReference); await orasClient.PullImageAsync(outputFolder); @@ -195,29 +192,27 @@ private void ClearFolder(string directory) folder.Delete(true); } - private void OrasExecution(string command) + private async Task PushOneLayerImageAsync() { - Process process = new Process + string command = $"push {_testOneLayerImageReference} {_baseLayerTemplatePath}"; + try { - StartInfo = new ProcessStartInfo(Path.Combine(AppContext.BaseDirectory, "oras.exe")), - }; - - process.StartInfo.Arguments = command; - process.StartInfo.RedirectStandardError = true; - process.EnableRaisingEvents = true; - process.Start(); + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + } + catch + { + _isOrasValid = false; + } + } - StreamReader errStreamReader = process.StandardError; - process.WaitForExit(30000); - if (process.HasExited) + private async Task PushMultiLayersImageAsync() + { + string command = $"push {_testMultiLayersImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}"; + try { - var error = errStreamReader.ReadToEnd(); - if (!string.IsNullOrEmpty(error)) - { - _isOrasValid = false; - } + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); } - else + catch { _isOrasValid = false; } diff --git a/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/OCIFileManagerTests.cs b/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/OCIFileManagerTests.cs index 3b94fcf10..136e65140 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/OCIFileManagerTests.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement.UnitTests/OCIFileManagerTests.cs @@ -5,16 +5,16 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Threading.Tasks; +using Microsoft.Health.Fhir.TemplateManagement.Client; using Microsoft.Health.Fhir.TemplateManagement.Exceptions; using Xunit; namespace Microsoft.Health.Fhir.TemplateManagement.UnitTests { - public class OCIFileManagerTests + public class OCIFileManagerTests : IAsyncLifetime { private readonly string _containerRegistryServer; private readonly string _baseLayerTemplatePath = "TestData/TarGzFiles/layer1.tar.gz"; @@ -28,20 +28,17 @@ public OCIFileManagerTests() _containerRegistryServer = Environment.GetEnvironmentVariable("TestContainerRegistryServer"); _testOneLayerImageReference = _containerRegistryServer + "/templatetest:user1"; _testMultiLayersImageReference = _containerRegistryServer + "/templatetest:user2"; - PushOneLayerImage(); - PushMultiLayersImage(); } - private void PushOneLayerImage() + public async Task InitializeAsync() { - string command = $"push {_testOneLayerImageReference} {_baseLayerTemplatePath}"; - OrasExecution(command); + await PushOneLayerImageAsync(); + await PushMultiLayersImageAsync(); } - private void PushMultiLayersImage() + public Task DisposeAsync() { - string command = $"push {_testMultiLayersImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}"; - OrasExecution(command); + return Task.CompletedTask; } public static IEnumerable GetValidOutputFolder() @@ -157,43 +154,41 @@ public async Task GivenAnImageReferenceAndInputFolder_WhenPushOCIFiles_CorrectIm Assert.Null(ex); } - private void OrasExecution(string command) + private void ClearFolder(string directory) { - Process process = new Process + if (!Directory.Exists(directory)) { - StartInfo = new ProcessStartInfo(Path.Combine(AppContext.BaseDirectory, "oras.exe")), - }; + return; + } - process.StartInfo.Arguments = command; - process.StartInfo.RedirectStandardError = true; - process.EnableRaisingEvents = true; - process.Start(); + DirectoryInfo folder = new DirectoryInfo(directory); + folder.Delete(true); + } - StreamReader errStreamReader = process.StandardError; - process.WaitForExit(30000); - if (process.HasExited) + private async Task PushOneLayerImageAsync() + { + string command = $"push {_testOneLayerImageReference} {_baseLayerTemplatePath}"; + try { - var error = errStreamReader.ReadToEnd(); - if (!string.IsNullOrEmpty(error)) - { - _isOrasValid = false; - } + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); } - else + catch { _isOrasValid = false; } } - private void ClearFolder(string directory) + private async Task PushMultiLayersImageAsync() { - if (!Directory.Exists(directory)) + string command = $"push {_testMultiLayersImageReference} {_baseLayerTemplatePath} {_userLayerTemplatePath}"; + try { - return; + await OrasClient.OrasExecutionAsync(command, Directory.GetCurrentDirectory()); + } + catch + { + _isOrasValid = false; } - - DirectoryInfo folder = new DirectoryInfo(directory); - folder.Delete(true); } } } diff --git a/src/Microsoft.Health.Fhir.TemplateManagement/Client/OrasClient.cs b/src/Microsoft.Health.Fhir.TemplateManagement/Client/OrasClient.cs index b37767187..a6418f1a8 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement/Client/OrasClient.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement/Client/OrasClient.cs @@ -52,12 +52,13 @@ public async Task PushImageAsync(string inputFolder) await OrasExecutionAsync(string.Concat(command, argument), inputFolder); } - private async Task OrasExecutionAsync(string command, string orasWorkingDirectory) + public static async Task OrasExecutionAsync(string command, string orasWorkingDirectory) { TaskCompletionSource eventHandled = new TaskCompletionSource(); + Process process = new Process { - StartInfo = new ProcessStartInfo(Path.Combine(AppContext.BaseDirectory, "oras.exe")), + StartInfo = new ProcessStartInfo(Constants.OrasFile), }; process.StartInfo.Arguments = command; diff --git a/src/Microsoft.Health.Fhir.TemplateManagement/Constants.cs b/src/Microsoft.Health.Fhir.TemplateManagement/Constants.cs index 611b65eb8..071cb1a8d 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement/Constants.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement/Constants.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Fhir.TemplateManagement { internal static class Constants { - internal const string DefaultTemplatePath = "DefaultTemplates.tar.gz"; + internal const string DefaultTemplatePath = "Hl7v2DefaultTemplates.tar.gz"; // Accept meidia type for manifest. internal const string MediatypeV2Manifest = "application/vnd.docker.distribution.manifest.v2+json"; @@ -27,5 +27,7 @@ internal static class Constants internal const string OverlayMetaJsonFile = ".image/meta.info"; internal const int TimeOutMilliseconds = 30000; + + internal const string OrasFile = "oras"; } } diff --git a/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.Liquid.Converter.nuspec b/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.Liquid.Converter.nuspec index 65d2f929c..cfa14d16f 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.Liquid.Converter.nuspec +++ b/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.Liquid.Converter.nuspec @@ -27,12 +27,12 @@ - + - + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.TemplateManagement.csproj b/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.TemplateManagement.csproj index d5cce43a1..6a38e53cb 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.TemplateManagement.csproj +++ b/src/Microsoft.Health.Fhir.TemplateManagement/Microsoft.Health.Fhir.TemplateManagement.csproj @@ -5,13 +5,8 @@ netcoreapp3.1 true Microsoft.Health.Fhir.Liquid.Converter.nuspec + https://github.com/deislabs/oras/releases/download/v0.8.1/oras_0.8.1_windows_amd64.tar.gz - - - - PreserveNewest - - @@ -32,13 +27,25 @@ - - DefaultTemplates.tar.gz + + Hl7v2DefaultTemplates.tar.gz + + + + PreserveNewest + + - + + + + + + + + - \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.TemplateManagement/TemplateLayerParser.cs b/src/Microsoft.Health.Fhir.TemplateManagement/TemplateLayerParser.cs index e20b8a01c..6e3c4dc17 100644 --- a/src/Microsoft.Health.Fhir.TemplateManagement/TemplateLayerParser.cs +++ b/src/Microsoft.Health.Fhir.TemplateManagement/TemplateLayerParser.cs @@ -36,7 +36,7 @@ public static Dictionary ParseToTemplates(Dictionary item.Key, - item => item.Value == null ? null : Encoding.UTF8.GetString(item.Value)); + item => item.Value == null ? null : GetContentWithoutBOM(item.Value)); var parsedTemplate = TemplateUtility.ParseHl7v2Templates(fileContent); return parsedTemplate; } @@ -45,5 +45,13 @@ public static Dictionary ParseToTemplates(Dictionary