From c6a87f8f15bf9bb9b2c93b5d1d82803ea2dccfef Mon Sep 17 00:00:00 2001 From: Qiwei Jin Date: Fri, 18 Dec 2020 21:47:23 +0800 Subject: [PATCH] hot fix (#155) * Fix command-line options (#132) * [DotLiquid] Refine template loading performance (#138) * Refine template loading * Copy list of dictionary in template provider * Add TemplateLocalFileSystem * Fix typo * fix bug for pull templates CLI (#145) * update readme (#135) * update readme * update readme * update readme * update doc * refine CLI options and fix typo (#144) * fix typo and refine commandline description * fix typo * [DotLiquid] Add description for "-t" option (#147) * Fixed FHIR server name (#143) * Add description for -t * Remove extra period * Remove unused sentence * Refine template zip name Co-authored-by: Ranvijay Kumar * Fixed FHIR server name (#143) (#150) Co-authored-by: Ranvijay Kumar Co-authored-by: Boya Wu <38548227+BoyaWu10@users.noreply.github.com> Co-authored-by: sowu880 <57981365+sowu880@users.noreply.github.com> Co-authored-by: Ranvijay Kumar --- README.md | 30 ++++- .../Models/PullTemplateOptions.cs | 5 +- .../Models/PushTemplateOptions.cs | 5 +- .../Program.cs | 12 +- .../TemplateManagementLogicHandler.cs | 1 + .../DotLiquids/EvaluateTests.cs | 22 ++-- .../DotLiquids/MemoryFileSystemTests.cs | 44 ++++--- .../TemplateLocalFileSystemTests.cs | 56 ++++++++ .../Hl7v2/Hl7v2TemplateProviderTests.cs | 17 ++- .../DotLiquids/MemoryFileSystem.cs | 18 ++- .../DotLiquids/TemplateLocalFileSystem.cs | 124 ++++++++++++++++++ .../Hl7v2/Hl7v2Processor.cs | 2 +- .../Hl7v2/Hl7v2TemplateProvider.cs | 51 ++----- .../IFhirConverterTemplateFileSystem.cs | 15 +++ .../ITemplateProvider.cs | 8 +- .../Utilities/TemplateUtility.cs | 4 +- .../Models/TemplateLayer.cs | 4 +- 17 files changed, 307 insertions(+), 111 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/TemplateLocalFileSystem.cs create mode 100644 src/Microsoft.Health.Fhir.Liquid.Converter/IFhirConverterTemplateFileSystem.cs diff --git a/README.md b/README.md index 6e2e0747b..5cbe36436 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ FHIR Converter with DotLiquid engine is integrated into the [FHIR Server for Azu This project consists of the following components: -1. A command-line tool for converting data. +1. A command-line tool for converting data and managing templates. 2. [Templates](data/Templates) for HL7 v2 to FHIR conversion. 3. [Sample data](data/SampleData) for testing purpose. @@ -41,24 +41,46 @@ FHIR Converter is integrated into the FHIR Server for Azure to run as part of th ### Command-line tool +**Convert Data** + The command-line tool can be used to convert a folder containing HL7 v2 messages to FHIR resources. Here are the parameters that the tool accepts: | Option | Name | Optionality | Default | Description | | ----- | ----- | ----- |----- |----- | | -d | TemplateDirectory | Required | | Root directory of templates. | -| -r | RootTemplate | Required | | Name of root template. | +| -r | RootTemplate | Required | | Name of root template. Valid values are ADT_A01, OML_O21, ORU_R01, VXU_V04. | | -c | InputDataContent | Optional| | Input data content. Specify OutputDataFile to get the results. | | -f | OutputDataFile | Optional | | Output data file. | -| -i | InputDataFolder | Optional | | Input data folder. Specify OutputDataFolder to get the results.. | +| -i | InputDataFolder | Optional | | Input data folder. Specify OutputDataFolder to get the results. | | -o | OutputDataFolder | Optional | | Output data folder. | +| -t | IsTraceInfo | Optional | | Provide trace information in the output if "-t" is set. | | --version | Version | Optional | | Display version information. | | --help | Help | Optional | | Display usage information of this tool. | Example usage to convert HL7 v2 messages to FHIR resources in a folder: ``` ->.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe -d myTemplateDirectory -r ADT_A01 -i myInputDataFolder -o myOutputDataFolder +>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe convert -d myTemplateDirectory -r ADT_A01 -i myInputDataFolder -o myOutputDataFolder +``` + +**Manage Templates** + +The command-line tool also supports managing different versions of templates from Azure Container Registry (ACR). Users can customize templates and store them on ACR if default templates can not meet requirements. After [ACR authentication](docs/TemplateManagementCLI.md), users can pull and push templates from/to a remote ACR through our tool. + +Example command to push a collection of templates to ACR image from a folder: +``` +>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe push testacr.azurecr.io/templatetest:default myInputFolder ``` +Example usage of pulling an image of templates in a folder: + +``` +>.\Microsoft.Health.Fhir.Liquid.Converter.Tool.exe pull testacr.azurecr.io/templatetest@sha256:412ea84f1bb1a9d98345efb7b427ba89616ec29ac332d543eff9a2161ca12a58 myOutputFolder + +``` +More details of usage are given in [Template Management CLI tool](docs/TemplateManagementCLI.md). + +Besides current version of [templates](data/Templates) given in our project, other versions that released by Microsoft are stored in a public ACR: healthplatformregistry.azurecr.io, users can directly pull templates from ``` healthplatformregistry.azurecr.io/hl7v2defaulttemplates: ``` without authentication. +>Note!: Template version is aligned with the version of FHIR Converter. ### A note on Resource ID generation diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs index 195efc991..ac7ab71d9 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PullTemplateOptions.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models { - [Verb("pull", HelpText = "Pull template image to registry")] + [Verb("pull", HelpText = "Pull a template image from a registry")] public class PullTemplateOptions { [Value(0, Required = true, HelpText = "Image reference: /: or /@")] @@ -18,8 +18,5 @@ public class PullTemplateOptions [Option('f', "ForceOverride", Required = false, Default = false, HelpText = "Force to override existed files")] public bool ForceOverride { get; set; } - - [Option('e', "ErrorJsonFile", Required = false, Default = null, HelpText = "Output error message File.")] - public string ErrorJsonFile { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs index e5b3071e5..bae5c354d 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Models/PushTemplateOptions.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Tool.Models { - [Verb("push", HelpText = "Push template image to registry")] + [Verb("push", HelpText = "Push a template image to a registry")] public class PushTemplateOptions { [Value(0, Required = true, HelpText = "Image reference: /:")] @@ -18,8 +18,5 @@ public class PushTemplateOptions [Option('n', "NewBaseLayer", Required = false, Default = false, HelpText = "Build new base layer")] public bool BuildNewBaseLayer { get; set; } - - [Option('e', "ErrorJsonFile", Required = false, Default = null, HelpText = "Output error message File.")] - public string ErrorJsonFile { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs index c318f82c1..9a1922b12 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/Program.cs @@ -4,9 +4,9 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Threading.Tasks; using CommandLine; -using CommandLine.Text; using Microsoft.Health.Fhir.Liquid.Converter.Tool.Models; namespace Microsoft.Health.Fhir.Liquid.Converter.Tool @@ -21,7 +21,7 @@ public static async Task Main(string[] args) parseResult.WithParsed(options => ConverterLogicHandler.Convert(options)); await parseResult.WithParsedAsync(options => TemplateManagementLogicHandler.PullAsync(options)); await parseResult.WithParsedAsync(options => TemplateManagementLogicHandler.PushAsync(options)); - parseResult.WithNotParsed((errors) => HandleOptionsParseError(parseResult)); + parseResult.WithNotParsed((errors) => HandleOptionsParseError(errors)); return 0; } catch (Exception ex) @@ -31,10 +31,12 @@ public static async Task Main(string[] args) } } - private static void HandleOptionsParseError(ParserResult parseResult) + private static void HandleOptionsParseError(IEnumerable errors) { - var usageText = HelpText.RenderUsageText(parseResult); - throw new InputParameterException(usageText); + if (!errors.IsHelp() && !errors.IsVersion()) + { + throw new InputParameterException(@"The input option is invalid."); + } } } } \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs index 1db82577b..08c7faddb 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.Tool/TemplateManagementLogicHandler.cs @@ -25,6 +25,7 @@ internal static async Task PullAsync(PullTemplateOptions options) OCIFileManager fileManager = new OCIFileManager(options.ImageReference, options.OutputTemplateFolder); await fileManager.PullOCIImageAsync(); + fileManager.UnpackOCIImage(); Console.WriteLine($"Successfully pulled templates to {options.OutputTemplateFolder} folder"); } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs index cafb33701..a2a341c6e 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/EvaluateTests.cs @@ -7,13 +7,17 @@ using System.Globalization; using DotLiquid; using DotLiquid.Exceptions; +using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; +using Microsoft.Health.Fhir.Liquid.Converter.Utilities; using Xunit; namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids { public class EvaluateTests { + public const string TemplateName = "TemplateName"; + public static IEnumerable GetValidEvaluateTemplateContents() { yield return new object[] { @"{% evaluate bundleId using 'ID/Bundle' -%}" }; @@ -41,15 +45,15 @@ public static IEnumerable GetInvalidEvaluateTemplateContents() public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultShouldBeReturned(string templateContent) { // Template should be parsed correctly - var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); - var template = Template.Parse(templateContent); + var template = TemplateUtility.ParseTemplate(TemplateName, templateContent); Assert.True(template.Root.NodeList.Count > 0); // Template should be rendered correctly + var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); var context = new Context( environments: new List(), outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider }), + registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: 0, @@ -62,17 +66,14 @@ public void GivenValidEvaluateTemplateContent_WhenParseAndRender_CorrectResultSh [MemberData(nameof(GetInvalidEvaluateTemplateContents))] public void GivenInvalidEvaluateTemplateContent_WhenParse_ExceptionsShouldBeThrown(string templateContent) { - var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); - Assert.Throws(() => Template.Parse(templateContent)); + Assert.Throws(() => TemplateUtility.ParseTemplate(TemplateName, templateContent)); } [Fact] public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown() { - var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); - // No template file system - var template = Template.Parse(@"{% evaluate bundleId using 'ID/Bundle' Data: hl7v2Data -%}"); + var template = TemplateUtility.ParseTemplate(TemplateName, @"{% evaluate bundleId using 'ID/Bundle' Data: hl7v2Data -%}"); var context = new Context( environments: new List(), outerScope: new Hash(), @@ -84,11 +85,12 @@ public void GivenInvalidSnippet_WhenRender_ExceptionsShouldBeThrown() Assert.Throws(() => template.Render(RenderParameters.FromContext(context, CultureInfo.InvariantCulture))); // Valid template file system but no such template - template = Template.Parse(@"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}"); + template = TemplateUtility.ParseTemplate(TemplateName, @"{% evaluate bundleId using 'ID/Foo' Data: hl7v2Data -%}"); + var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); context = new Context( environments: new List(), outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider }), + registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: 0, diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs index fd0c8174f..3f6646608 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/MemoryFileSystemTests.cs @@ -7,8 +7,8 @@ using System.Collections.Generic; using System.Globalization; using DotLiquid; +using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; -using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; using Xunit; namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids @@ -26,9 +26,11 @@ public void GivenAValidTemplateCollection_WhenGetTemplate_CorrectResultShouldBeR }, }; - var templateProvider = new Hl7v2TemplateProvider(templateCollection); - Assert.Equal("hello world", templateProvider.GetTemplate("template1").Render()); - Assert.Null(templateProvider.GetTemplate("template2")); + var memoryFileSystem = new MemoryFileSystem(templateCollection); + Assert.Equal("hello world", memoryFileSystem.GetTemplate("template1").Render()); + Assert.Null(memoryFileSystem.GetTemplate("template2")); + Assert.Null(memoryFileSystem.GetTemplate(null)); + Assert.Null(memoryFileSystem.GetTemplate(string.Empty)); } [Fact] @@ -55,11 +57,11 @@ public void GivenTwoValidTemplateCollection_WhenGetTemplate_CorrectResultShouldB }, }; - var templateProvider = new Hl7v2TemplateProvider(templateCollection); - Assert.Null(templateProvider.GetTemplate("template1")); - Assert.Equal("template2 updated in customized layer", templateProvider.GetTemplate("template2").Render()); - Assert.Equal("template3 added in base layer", templateProvider.GetTemplate("template3").Render()); - Assert.Equal("template4 added in customized layer", templateProvider.GetTemplate("template4").Render()); + var memoryFileSystem = new MemoryFileSystem(templateCollection); + Assert.Null(memoryFileSystem.GetTemplate("template1")); + Assert.Equal("template2 updated in customized layer", memoryFileSystem.GetTemplate("template2").Render()); + Assert.Equal("template3 added in base layer", memoryFileSystem.GetTemplate("template3").Render()); + Assert.Equal("template4 added in customized layer", memoryFileSystem.GetTemplate("template4").Render()); } [Fact] @@ -73,13 +75,15 @@ public void GivenAValidTemplateCollection_WhenGetTemplateWithContext_CorrectResu }, }; - var templateProvider = new Hl7v2TemplateProvider(templateCollection); + var memoryFileSystem = new MemoryFileSystem(templateCollection); var context = new Context(CultureInfo.InvariantCulture); context["template1"] = "template1"; context["template2"] = "template2"; - Assert.Equal("hello world", templateProvider.GetTemplate(context, "template1").Render()); - Assert.Throws(() => templateProvider.GetTemplate(context, "template2")); - Assert.Throws(() => templateProvider.GetTemplate(context, "template3")); + Assert.Equal("hello world", memoryFileSystem.GetTemplate(context, "template1").Render()); + Assert.Throws(() => memoryFileSystem.GetTemplate(context, "template2")); + Assert.Throws(() => memoryFileSystem.GetTemplate(context, "template3")); + Assert.Throws(() => memoryFileSystem.GetTemplate(context, null)); + Assert.Throws(() => memoryFileSystem.GetTemplate(context, string.Empty)); } [Fact] @@ -106,17 +110,17 @@ public void GivenTwoValidtTemplateCollection_WhenGetTemplateWithContext_CorrectR }, }; - var templateProvider = new Hl7v2TemplateProvider(templateCollection); + var memoryFileSystem = new MemoryFileSystem(templateCollection); var context = new Context(CultureInfo.InvariantCulture); context["'folder/template1'"] = "folder/template1"; context["template2"] = "template2"; context["template3"] = "template3"; context["template4"] = "template4"; - Assert.Throws(() => templateProvider.GetTemplate(context, "'folder/template1'")); - Assert.Equal("template2 updated in customized layer", templateProvider.GetTemplate(context, "template2").Render()); - Assert.Equal("template3 added in base layer", templateProvider.GetTemplate(context, "template3").Render()); - Assert.Equal("template4 added in customized layer", templateProvider.GetTemplate(context, "template4").Render()); + Assert.Throws(() => memoryFileSystem.GetTemplate(context, "'folder/template1'")); + Assert.Equal("template2 updated in customized layer", memoryFileSystem.GetTemplate(context, "template2").Render()); + Assert.Equal("template3 added in base layer", memoryFileSystem.GetTemplate(context, "template3").Render()); + Assert.Equal("template4 added in customized layer", memoryFileSystem.GetTemplate(context, "template4").Render()); } [Fact] @@ -130,10 +134,10 @@ public void GivenAValidTemplateCollection_WhenReadTemplateWithContext_ExceptionS }, }; - var templateProvider = new Hl7v2TemplateProvider(templateCollection); + var memoryFileSystem = new MemoryFileSystem(templateCollection); var context = new Context(CultureInfo.InvariantCulture); context["hello"] = "hello"; - Assert.Throws(() => templateProvider.ReadTemplateFile(context, "hello")); + Assert.Throws(() => memoryFileSystem.ReadTemplateFile(context, "hello")); } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs new file mode 100644 index 000000000..87216466a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/DotLiquids/TemplateLocalFileSystemTests.cs @@ -0,0 +1,56 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Globalization; +using DotLiquid; +using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids; +using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; +using Microsoft.Health.Fhir.Liquid.Converter.Models; +using Xunit; + +namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.DotLiquids +{ + public class TemplateLocalFileSystemTests + { + [Fact] + public void GivenAValidTemplateDirectory_WhenGetTemplate_CorrectResultsShouldBeReturned() + { + var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2); + var context = new Context(CultureInfo.InvariantCulture); + + // Template exists + Assert.NotNull(templateLocalFileSystem.GetTemplate("ADT_A01")); + + // Template does not exist + Assert.Null(templateLocalFileSystem.GetTemplate("Foo")); + } + + [Fact] + public void GivenAValidTemplateDirectory_WhenGetTemplateWithContext_CorrectResultsShouldBeReturned() + { + var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2); + var context = new Context(CultureInfo.InvariantCulture); + + // Template exists + context["ADT_A01"] = "ADT_A01"; + Assert.NotNull(templateLocalFileSystem.GetTemplate(context, "ADT_A01")); + + // Template does not exist + context["Foo"] = "Foo"; + Assert.Throws(() => templateLocalFileSystem.GetTemplate(context, "Foo")); + Assert.Throws(() => templateLocalFileSystem.GetTemplate(context, "Bar")); + } + + [Fact] + public void GivenAValidTemplateDirectory_WhenReadTemplateWithContext_ExceptionShouldBeThrown() + { + var templateLocalFileSystem = new TemplateLocalFileSystem(Constants.Hl7v2TemplateDirectory, DataType.Hl7v2); + var context = new Context(CultureInfo.InvariantCulture); + context["ADT_A01"] = "ADT_A01"; + Assert.Throws(() => templateLocalFileSystem.ReadTemplateFile(context, "hello")); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2TemplateProviderTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2TemplateProviderTests.cs index cdcdf7fe6..6ec1f5a1b 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2TemplateProviderTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Hl7v2/Hl7v2TemplateProviderTests.cs @@ -3,10 +3,11 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; using System.IO; +using DotLiquid; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Hl7v2; -using Microsoft.Health.Fhir.Liquid.Converter.Models; using Xunit; namespace Microsoft.Health.Fhir.Liquid.Converter.UnitTests.Hl7v2 @@ -16,17 +17,15 @@ public class Hl7v2TemplateProviderTests [Fact] public void GivenATemplateDirectory_WhenLoadTemplates_CorrectResultsShouldBeReturned() { + // Valid template directory var templateProvider = new Hl7v2TemplateProvider(Constants.Hl7v2TemplateDirectory); - Assert.True(templateProvider.GetTemplate("ADT_A01").Root.NodeList.Count > 0); - Assert.Throws(() => templateProvider.LoadTemplates(null)); - Assert.Throws(() => templateProvider.LoadTemplates(string.Empty)); - Assert.Throws(() => templateProvider.LoadTemplates(Path.Join("a", "b", "c"))); + // Invalid template directory + Assert.Throws(() => new Hl7v2TemplateProvider(string.Empty)); + Assert.Throws(() => new Hl7v2TemplateProvider(Path.Join("a", "b", "c"))); - var exception = Assert.Throws(() => templateProvider.LoadTemplates(@"TestTemplates")); - Assert.Equal(FhirConverterErrorCode.TemplateLoadingError, exception.FhirConverterErrorCode); - var innerException = exception.InnerException as FhirConverterException; - Assert.Equal(FhirConverterErrorCode.TemplateSyntaxError, innerException.FhirConverterErrorCode); + // Template collection + templateProvider = new Hl7v2TemplateProvider(new List>()); } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/MemoryFileSystem.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/MemoryFileSystem.cs index 7e576aa1f..5d5a90860 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/MemoryFileSystem.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/MemoryFileSystem.cs @@ -6,15 +6,23 @@ using System; using System.Collections.Generic; using DotLiquid; -using DotLiquid.FileSystems; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Models; namespace Microsoft.Health.Fhir.Liquid.Converter.DotLiquids { - public class MemoryFileSystem : ITemplateFileSystem + public class MemoryFileSystem : IFhirConverterTemplateFileSystem { - protected List> TemplateCollection { get; set; } = new List>(); + private readonly List> _templateCollection; + + public MemoryFileSystem(List> templateCollection) + { + _templateCollection = new List>(); + foreach (var templates in templateCollection) + { + _templateCollection.Add(new Dictionary(templates)); + } + } public string ReadTemplateFile(Context context, string templateName) { @@ -26,7 +34,7 @@ public Template GetTemplate(Context context, string templateName) var templatePath = (string)context[templateName]; if (templatePath == null) { - throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templatePath)); + throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templateName)); } return GetTemplate(templatePath) ?? throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templatePath)); @@ -39,7 +47,7 @@ public Template GetTemplate(string templateName) return null; } - foreach (var templates in TemplateCollection) + foreach (var templates in _templateCollection) { if (templates != null && templates.ContainsKey(templateName)) { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/TemplateLocalFileSystem.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/TemplateLocalFileSystem.cs new file mode 100644 index 000000000..b0aa6e9a3 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/DotLiquids/TemplateLocalFileSystem.cs @@ -0,0 +1,124 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.IO; +using DotLiquid; +using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; +using Microsoft.Health.Fhir.Liquid.Converter.Models; +using Microsoft.Health.Fhir.Liquid.Converter.Utilities; + +namespace Microsoft.Health.Fhir.Liquid.Converter.DotLiquids +{ + public class TemplateLocalFileSystem : IFhirConverterTemplateFileSystem + { + private readonly string _templateDirectory; + + private Dictionary _templateCache; + + public TemplateLocalFileSystem(string templateDirectory, DataType dataType) + { + if (!Directory.Exists(templateDirectory)) + { + throw new ConverterInitializeException(FhirConverterErrorCode.TemplateFolderNotFound, string.Format(Resources.TemplateFolderNotFound, templateDirectory)); + } + + _templateDirectory = templateDirectory; + _templateCache = new Dictionary(); + + if (dataType == DataType.Hl7v2) + { + LoadCodeSystemMappingTemplate(); + } + } + + public string ReadTemplateFile(Context context, string templateName) + { + throw new NotImplementedException(); + } + + public Template GetTemplate(Context context, string templateName) + { + var templatePath = (string)context[templateName]; + if (templatePath == null) + { + throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templateName)); + } + + return GetTemplate(templatePath) ?? throw new RenderException(FhirConverterErrorCode.TemplateNotFound, string.Format(Resources.TemplateNotFound, templatePath)); + } + + public Template GetTemplate(string templateName) + { + if (string.IsNullOrEmpty(templateName)) + { + return null; + } + + // Get template from cache first + if (_templateCache.ContainsKey(templateName)) + { + return _templateCache[templateName]; + } + + // If not cached, search local file system + var templatePath = GetAbsoluteTemplatePath(templateName); + if (File.Exists(templatePath)) + { + var templateContent = LoadTemplate(templatePath); + var template = TemplateUtility.ParseTemplate(templateName, templateContent); + _templateCache[templateName] = template; + return template; + } + + return null; + } + + private void LoadCodeSystemMappingTemplate() + { + var codeSystemMappingPath = Path.Join(_templateDirectory, "CodeSystem", "CodeSystem.json"); + if (File.Exists(codeSystemMappingPath)) + { + var content = LoadTemplate(codeSystemMappingPath); + var template = TemplateUtility.ParseCodeSystemMapping(content); + _templateCache["CodeSystem/CodeSystem"] = template; + } + } + + private string LoadTemplate(string templatePath) + { + try + { + return File.ReadAllText(templatePath); + } + catch (Exception ex) + { + throw new ConverterInitializeException(FhirConverterErrorCode.TemplateLoadingError, string.Format(Resources.TemplateLoadingError, ex.Message), ex); + } + } + + private string GetAbsoluteTemplatePath(string templateName) + { + var result = _templateDirectory; + var pathSegments = templateName.Split(Path.AltDirectorySeparatorChar); + + // Root templates + if (pathSegments.Length == 1) + { + return Path.Join(_templateDirectory, $"{pathSegments[0]}.liquid"); + } + + // Snippets + pathSegments[^1] = $"_{pathSegments[^1]}.liquid"; + foreach (var pathSegment in pathSegments) + { + result = Path.Join(result, pathSegment); + } + + return result; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs index 2f154a0dd..cf182d5eb 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2Processor.cs @@ -70,7 +70,7 @@ private Context CreateContext(ITemplateProvider templateProvider, Hl7v2Data hl7v var context = new Context( environments: new List() { Hash.FromAnonymousObject(new { hl7v2Data }) }, outerScope: new Hash(), - registers: Hash.FromAnonymousObject(new { file_system = templateProvider }), + registers: Hash.FromAnonymousObject(new { file_system = templateProvider.GetTemplateFileSystem() }), errorsOutputMode: ErrorsOutputMode.Rethrow, maxIterations: 0, timeout: timeout, diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2TemplateProvider.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2TemplateProvider.cs index 843737447..a5eb6b1c9 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2TemplateProvider.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Hl7v2/Hl7v2TemplateProvider.cs @@ -3,65 +3,36 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.IO; using DotLiquid; +using DotLiquid.FileSystems; using Microsoft.Health.Fhir.Liquid.Converter.DotLiquids; -using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Models; -using Microsoft.Health.Fhir.Liquid.Converter.Utilities; namespace Microsoft.Health.Fhir.Liquid.Converter.Hl7v2 { - public class Hl7v2TemplateProvider : MemoryFileSystem, ITemplateProvider + public class Hl7v2TemplateProvider : ITemplateProvider { - private const string TemplateFileExtension = ".liquid"; + private readonly IFhirConverterTemplateFileSystem _fileSystem; public Hl7v2TemplateProvider(string templateDirectory) { - TemplateCollection = LoadTemplates(templateDirectory); + _fileSystem = new TemplateLocalFileSystem(templateDirectory, DataType.Hl7v2); } public Hl7v2TemplateProvider(List> templateCollection) { - TemplateCollection = templateCollection; + _fileSystem = new MemoryFileSystem(templateCollection); } - public List> LoadTemplates(string templateDirectory) + public Template GetTemplate(string templateName) { - if (!Directory.Exists(templateDirectory)) - { - throw new ConverterInitializeException(FhirConverterErrorCode.TemplateFolderNotFound, string.Format(Resources.TemplateFolderNotFound, templateDirectory)); - } - - try - { - // Load DotLiquid templates - var templates = new Dictionary(); - var templatePaths = Directory.EnumerateFiles(templateDirectory, "*" + TemplateFileExtension, SearchOption.AllDirectories); - foreach (var templatePath in templatePaths) - { - var name = Path.GetRelativePath(templateDirectory, templatePath); - templates[name] = File.ReadAllText(templatePath); - } - - // Load code system mapping - var codeSystemMappingPath = Path.Join(templateDirectory, "CodeSystem", "CodeSystem.json"); - if (File.Exists(codeSystemMappingPath)) - { - var name = Path.GetRelativePath(templateDirectory, codeSystemMappingPath); - templates[name] = File.ReadAllText(codeSystemMappingPath); - } + return _fileSystem.GetTemplate(templateName); + } - // Parse templates - var parsedTemplates = TemplateUtility.ParseHl7v2Templates(templates); - return new List>() { parsedTemplates }; - } - catch (Exception ex) - { - throw new ConverterInitializeException(FhirConverterErrorCode.TemplateLoadingError, string.Format(Resources.TemplateLoadingError, ex.Message), ex); - } + public ITemplateFileSystem GetTemplateFileSystem() + { + return _fileSystem; } } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/IFhirConverterTemplateFileSystem.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/IFhirConverterTemplateFileSystem.cs new file mode 100644 index 000000000..793da8ae4 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/IFhirConverterTemplateFileSystem.cs @@ -0,0 +1,15 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using DotLiquid; +using DotLiquid.FileSystems; + +namespace Microsoft.Health.Fhir.Liquid.Converter +{ + public interface IFhirConverterTemplateFileSystem : ITemplateFileSystem + { + public Template GetTemplate(string templateName); + } +} diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/ITemplateProvider.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/ITemplateProvider.cs index b601eeb05..9b36ee159 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/ITemplateProvider.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/ITemplateProvider.cs @@ -3,17 +3,15 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Collections.Generic; using DotLiquid; +using DotLiquid.FileSystems; namespace Microsoft.Health.Fhir.Liquid.Converter { public interface ITemplateProvider { - public List> LoadTemplates(string templateDirectory); - - public Template GetTemplate(Context context, string templateName); - public Template GetTemplate(string templateName); + + public ITemplateFileSystem GetTemplateFileSystem(); } } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Utilities/TemplateUtility.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Utilities/TemplateUtility.cs index 9cf653d9b..9e6f54672 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Utilities/TemplateUtility.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Utilities/TemplateUtility.cs @@ -48,7 +48,7 @@ public static Dictionary ParseHl7v2Templates(Dictionary