DYN-6569 Extended Characters Md2HTML Fix (#15377)

src/DynamoUtilities/CLIWrapper.cs

@@ -3,6 +3,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.Reflection;
+using System.Text;
 using System.Threading.Tasks;
 
 namespace Dynamo.Utilities
@@ -37,7 +38,8 @@ protected virtual void StartProcess(string relativeEXEPath, string argString)
                 RedirectStandardOutput = true,
                 RedirectStandardInput = true,
                 RedirectStandardError = true,
-
+                StandardInputEncoding = Encoding.UTF8,
+                StandardOutputEncoding = Encoding.UTF8,
                 UseShellExecute = false,
                 Arguments = argString,
                 FileName = GetToolPath(relativeEXEPath)

src/Tools/Md2Html/Program.cs

@@ -24,16 +24,15 @@ static void Main(string[] args)
             try
             {
                 Console.WriteLine("");
-
                 while (true)
                 {
                     var line = Console.ReadLine();
-                    if (line == @"<<<<<Sanitize>>>>>")
+                    if (line.Contains(@"<<<<<Sanitize>>>>>"))
                     {
                         Console.WriteLine(@"<<<<<Sod>>>>>");
                         Sanitize();
                     }
-                    else if (line == @"<<<<<Convert>>>>>")
+                    else if (line.Contains(@"<<<<<Convert>>>>>"))
                     {
                         Console.WriteLine(@"<<<<<Sod>>>>>");
                         Convert();

test/DynamoCoreWpfTests/ViewExtensions/DocumentationBrowserViewExtensionTests.cs

@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Reflection;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Windows;
 using System.Windows.Controls;
@@ -994,6 +995,61 @@ public void CheckThatEmbeddedMdContentDoesNotSanitize(string resource)
             }
         }
 
+        [Test]
+        [TestCase("en-US")]
+        [TestCase("cs-CZ")]
+        [TestCase("ko-KR")]
+        public void CheckThatExtendedCharactersAreCorrectlyInHTML(string language)
+        {
+            var testDirectory = GetTestDirectory(ExecutingDirectory);
+            string mdFile = "Autodesk.DesignScript.Geometry.Curve.SweepAsSurface.md";
+            string resource = Path.Combine(testDirectory, "Tools", "docGeneratorTestFiles", "fallback_docs", language, mdFile);
+
+            Assert.True(File.Exists(resource), "The resource provided {0} doesn't exist in the path {1}", mdFile, resource);
+
+            // Arrange
+            var content = File.ReadAllText(resource, Encoding.UTF8);
+
+            using (var converter = new Md2Html())
+            {
+                // Act
+                var html = converter.ParseMd2Html(content, ExecutingDirectory);
+                var output = converter.SanitizeHtml(html);
+
+                // Assert
+                Assert.IsTrue(string.IsNullOrEmpty(output));
+
+                Regex rxExp = new Regex(@"#+\s[^\n]*\n(.*?)(?=\n##?\s|$)", RegexOptions.Singleline);
+
+                //Apply RegEx expression to the md file to getting the content without headers
+                MatchCollection matches = rxExp.Matches(content);
+                foreach (Match match in matches)
+                {
+                    if (match.Groups.Count == 0) continue;
+
+                    var UTF8Content = match.Groups[1].Value.Trim();
+
+                    //Discard the image due that inside the html is converted to <img .......
+                    if(!UTF8Content.StartsWith("![")) 
+                    {
+                        // Assert
+                        //Clean the content to remove characters also removed by the Md2Html.exe tool
+                        var cleanedContent = UTF8Content.Replace("___", string.Empty).Trim();
+
+                        //Apply a Regular Expresion in the HTML file for getting the first <p> 
+                        Match m = Regex.Match(html, "<p>(.+?)<\\/p>", RegexOptions.IgnoreCase);
+                        string specificHTMLContent = string.Empty;
+                        if (m.Success)
+                        {
+                            specificHTMLContent = m.Groups[1].Value; 
+                        }
+
+                        Assert.IsTrue(specificHTMLContent == cleanedContent, "Part of the MD file content was not found in the HTML File");
+                    }                   
+                }
+            }
+        }
+
         #region Helpers
         private static string[] htmlResources()
         {
@@ -1013,6 +1069,12 @@ private static string[] getContentFiles(string wildcard)
             return Directory.GetFiles(docsDirectory, wildcard);
         }
 
+        public static string GetTestDirectory(string executingDirectory)
+        {
+            var directory = new DirectoryInfo(executingDirectory);
+            return Path.Combine(directory.Parent.Parent.Parent.FullName, "test");
+        }
+
         #endregion
     }
 }

test/Tools/docGeneratorTestFiles/fallback_docs/cs-CZ/Autodesk.DesignScript.Geometry.Curve.SweepAsSurface.md

@@ -0,0 +1,7 @@
+## Podrobnosti
+Uzel SweepAsSurface vytvoří povrch tažením vstupní křivky podél zadané trajektorie. V níže uvedeném příkladu vytvoříme křivku k tažení pomocí bloku kódu, který vytvoří tři body pro uzel Arc.ByThreePoints. Křivka trajektorie se vytvoří pomocí jednoduché čáry podél osy x. Uzel SweepAsSurface přesune křivku profilu podél křivky trajektorie, čímž vytvoří povrch.
+___
+## Vzorový soubor
+
+![SweepAsSurface](./Autodesk.DesignScript.Geometry.Curve.SweepAsSurface_img.jpg)
+

test/Tools/docGeneratorTestFiles/fallback_docs/en-US/Autodesk.DesignScript.Geometry.Curve.SweepAsSurface.md

@@ -0,0 +1,7 @@
+## In Depth
+SweepAsSurface will create a surface by sweeping an input curve along a specfied path. In the example below, we create a curve to sweep by useing a Code Block to create three points of an Arc.ByThreePoints node. A path curve is created a simple line along the x-axis. SweepAsSurface moves the profile curve along the path curve creating a surface.
+___
+## Example File
+
+![SweepAsSurface](./Autodesk.DesignScript.Geometry.Curve.SweepAsSurface_img.jpg)
+

test/Tools/docGeneratorTestFiles/fallback_docs/ko-KR/Autodesk.DesignScript.Geometry.Curve.SweepAsSurface.md

@@ -0,0 +1,7 @@
+## 상세
+SweepAsSurface는 지정된 경로를 따라 입력 곡선을 스윕하여 표면을 작성합니다. 아래 예에서는 Arc.ByThreePoints 노드의 세 점을 작성하기 위해 코드 블록을 사용하여 스윕할 곡선을 작성합니다. 경로 곡선은 X축을 따라 단순한 선으로 작성됩니다. SweepAsSurface는 경로 곡선을 따라 종단 곡선을 이동하여 표면을 작성합니다.
+___
+## 예제 파일
+
+![SweepAsSurface](./Autodesk.DesignScript.Geometry.Curve.SweepAsSurface_img.jpg)
+