New option to the JSON function to serialize unwrapped arrays (#2231)

Common request: ability of the JSON function to serialize [1,2,3] as `[1,2,3]` instead of `[{"Value":1},{"Value":2},{"Value":3}]`
microsoft · Mar 14, 2024 · d8d3e8c · d8d3e8c
1 parent d552c10
commit d8d3e8c
Show file tree

Hide file tree

Showing 11 changed files with 117 additions and 26 deletions.
diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs
@@ -778,6 +778,7 @@ internal static class TexlStrings
         public static ErrorResourceKey ErrJSONArg1UnsupportedType = new ErrorResourceKey("ErrJSONArg1UnsupportedType");
         public static ErrorResourceKey ErrJSONArg1ContainsUnsupportedMedia = new ErrorResourceKey("ErrJSONArg1ContainsUnsupportedMedia");
         public static ErrorResourceKey ErrJSONArg2IncompatibleOptions = new ErrorResourceKey("ErrJSONArg2IncompatibleOptions");
+        public static ErrorResourceKey ErrJSONArg2UnsupportedOption = new ErrorResourceKey("ErrJSONArg2UnsupportedOption");
         public static ErrorResourceKey ErrJSONArg1UnsupportedNestedType = new ErrorResourceKey("ErrJSONArg1UnsupportedNestedType");
         public static ErrorResourceKey ErrJSONArg1UnsupportedTypeWithNonBehavioral = new ErrorResourceKey("ErrJSONArg1UnsupportedTypeWithNonBehavioral");
         public static ErrorResourceKey ErrTraceInvalidCustomRecordType = new ErrorResourceKey("ErrTraceInvalidCustomRecordType");

diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs
@@ -17,9 +17,11 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins
     // JSON(data:any, [format:s])    
     internal class JsonFunction : BuiltinFunction
     {        
-        private const string _includeBinaryDataEnumValue = "B";
-        private const string _ignoreBinaryDataEnumValue = "G";
-        private const string _ignoreUnsupportedTypesEnumValue = "I";
+        private const char _includeBinaryDataEnumValue = 'B';
+        private const char _ignoreBinaryDataEnumValue = 'G';
+        private const char _ignoreUnsupportedTypesEnumValue = 'I';
+        private const char _flattenTableValuesEnumValue = '_';
+        private const char _indentFourEnumValue = '4';
 
         protected bool supportsLazyTypes = false;
 
@@ -120,9 +122,33 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[
 
                 if (nodeValue != null)
                 {
-                    ignoreUnsupportedTypes = nodeValue.Contains(_ignoreUnsupportedTypesEnumValue);
-                    includeBinaryData = nodeValue.Contains(_includeBinaryDataEnumValue);
-                    ignoreBinaryData = nodeValue.Contains(_ignoreBinaryDataEnumValue);                    
+                    foreach (var option in nodeValue)
+                    {
+                        switch (option)
+                        {
+                            case _ignoreBinaryDataEnumValue:
+                                ignoreBinaryData = true;
+                                break;
+                            case _ignoreUnsupportedTypesEnumValue:
+                                ignoreUnsupportedTypes = true;
+                                break;
+                            case _includeBinaryDataEnumValue:
+                                includeBinaryData = true;
+                                break;
+                            case _flattenTableValuesEnumValue:
+                            case _indentFourEnumValue:
+                                // Runtime-only options
+                                break;
+                            default:
+                                if (binding.Features.PowerFxV1CompatibilityRules)
+                                {
+                                    errors.EnsureError(optionsNode, TexlStrings.ErrJSONArg2UnsupportedOption, option);
+                                    return;
+                                }
+
+                                break;
+                        }
+                    }
 
                     if (includeBinaryData && ignoreBinaryData)
                     {

diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs b/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs
@@ -58,7 +58,7 @@ internal sealed class EnumStoreBuilder
                 },
                 {
                     LanguageConstants.JSONFormatEnumString,
-                    "%s[Compact:\"\", IndentFour:\"4\", IgnoreBinaryData:\"G\", IncludeBinaryData:\"B\", IgnoreUnsupportedTypes:\"I\"]"
+                    "%s[Compact:\"\", IndentFour:\"4\", IgnoreBinaryData:\"G\", IncludeBinaryData:\"B\", IgnoreUnsupportedTypes:\"I\", FlattenValueTables:\"_\"]"
                 },
                 { 
                     LanguageConstants.TraceSeverityEnumString,

diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs
@@ -56,7 +56,7 @@ internal FormulaValue Process()
 
                 using MemoryStream memoryStream = new MemoryStream();
                 using Utf8JsonWriter writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions() { Indented = flags.IndentFour, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping });
-                Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo);
+                Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo, flattenValueTables: flags.FlattenValueTables);
 
                 _arguments[0].Visit(jsonWriterVisitor);
                 writer.Flush();
@@ -89,6 +89,7 @@ private JsonFlags GetFlags()
                     flags.IgnoreUnsupportedTypes = arg1string.Value.Contains("I");
                     flags.IncludeBinaryData = arg1string.Value.Contains("B");
                     flags.IndentFour = arg1string.Value.Contains("4");
+                    flags.FlattenValueTables = arg1string.Value.Contains("_");
                 }
 
                 if (_arguments.Length > 1 && _arguments[1] is OptionSetValue arg1optionset)
@@ -97,6 +98,7 @@ private JsonFlags GetFlags()
                     flags.IgnoreUnsupportedTypes = arg1optionset.Option == "IgnoreUnsupportedTypes";
                     flags.IncludeBinaryData = arg1optionset.Option == "IncludeBinaryData";
                     flags.IndentFour = arg1optionset.Option == "IndentFour";
+                    flags.FlattenValueTables = arg1optionset.Option == "FlattenValueTables";
                 }
 
                 if ((flags.IncludeBinaryData && flags.IgnoreBinaryData) ||
@@ -113,13 +115,15 @@ private class Utf8JsonWriterVisitor : IValueVisitor
             {
                 private readonly Utf8JsonWriter _writer;
                 private readonly TimeZoneInfo _timeZoneInfo;
+                private readonly bool _flattenValueTables;
 
                 internal readonly List<ErrorValue> ErrorValues = new List<ErrorValue>();
 
-                internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo)
+                internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo, bool flattenValueTables)
                 {
                     _writer = writer;
                     _timeZoneInfo = timeZoneInfo;
+                    _flattenValueTables = flattenValueTables;
                 }
 
                 public void Visit(BlankValue blankValue)
@@ -254,6 +258,17 @@ public void Visit(TableValue tableValue)
                 {
                     _writer.WriteStartArray();
 
+                    var isSingleColumnValueTable = false;
+                    if (_flattenValueTables)
+                    {
+                        var fieldTypes = tableValue.Type.GetFieldTypes();
+                        var firstField = fieldTypes.FirstOrDefault();
+                        if (firstField != null && !fieldTypes.Skip(1).Any() && firstField.Name.Value == TexlFunction.ColumnName_ValueStr)
+                        {
+                            isSingleColumnValueTable = true;
+                        }
+                    }
+
                     foreach (DValue<RecordValue> row in tableValue.Rows)
                     {
                         if (row.IsBlank)
@@ -266,7 +281,15 @@ public void Visit(TableValue tableValue)
                         }
                         else
                         {
-                            row.Value.Visit(this);
+                            if (isSingleColumnValueTable)
+                            {
+                                var namedValue = row.Value.Fields.First();
+                                namedValue.Value.Visit(this);
+                            }
+                            else
+                            {
+                                row.Value.Visit(this);
+                            }
                         }
                     }
 
@@ -311,6 +334,8 @@ private class JsonFlags
                 internal bool IncludeBinaryData = false;
 
                 internal bool IndentFour = false;
+
+                internal bool FlattenValueTables = false;
             }
 
             private static DateTime ConvertToUTC(DateTime dateTime, TimeZoneInfo fromTimeZone)

diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx
@@ -4422,6 +4422,10 @@
     <value>The JSONFormat values '{0}' and '{1}' cannot be used together.</value>
     <comment>{Locked=JSONFormat} Error message shown to the user if they try to use incompatible values from the JSONFormat enumeration. The parameters are string values, members of that enumeration. For example: The JSONFormat values 'IgnoreBinaryData' and 'IncludeBinaryData' cannot be used together.</comment>
   </data>
+  <data name="ErrJSONArg2UnsupportedOption" xml:space="preserve">
+    <value>The value '{0}' is not supported as an option in the JSON function.</value>
+    <comment>{Locked=JSON} Error message shown to the user if they try to pass an option that is not supported to the JSON function. The parameter is a single-character string value. For example: The value '$' is not supported as an option in the JSON function.</comment>
+  </data>
   <data name="ErrJSONArg1ContainsUnsupportedMedia" xml:space="preserve">
     <value>The value passed to the JSON function contains media, and it is not supported by default. To allow JSON serialization of media values, make sure to use the IncludeBinaryData option in the 'format' parameter.</value>
     <comment>{Locked=JSON}{Locked=IncludeBinaryData}. Error message shown to the user if they try to serialize an object that contains media values, without specifying the flag that would allow that operation to happen. Media values are any form of media (audio, images, video) that are included as part of a record or a table. The term 'format' should be translated like the term 'JSONArg2' in this same file.</comment>

diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt
@@ -1,7 +1,4 @@
-// (UTC-08:00) Pacific Time (US & Canada)
-#SETUP: TimeZoneInfo("Pacific Standard Time")
-#SETUP: OptionSetTestSetup
-#SETUP: EnableJsonFunctions
+#SETUP: EnableJsonFunctions
 
 >> JSON()
 Errors: Error 0-6: Invalid number of arguments: received 0, expected 1-2.
@@ -53,9 +50,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'.
 >> JSON(Color.Turquoise)
 """#40e0d0ff"""
 
->> JSON(OptionSet.Option2)
-"""option_2"""
-
 >> JSON(TimeUnit.Hours)
 """hours"""
 
@@ -65,10 +59,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'.
 >> JSON(DateTimeValue("1970-01-01T00:00:00Z"))
 """1970-01-01T00:00:00.000Z"""
 
-// Midnight local time in Pacific Time is 8 AM UTC
->> JSON(DateTimeValue("1970-01-01T00:00:00"))
-"""1970-01-01T08:00:00.000Z"""
-
 // Independent from local timezone
 >> With({dt: DateTime(1987,6,5,4,30,0)}, JSON(DateAdd(dt,-TimeZoneOffset(dt),TimeUnit.Minutes), JSONFormat.IndentFour))
 """1987-06-05T04:30:00.000Z"""
@@ -94,10 +84,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'.
 >> JSON(DateTimeValue("2022-08-07T12:34:56Z"))
 """2022-08-07T12:34:56.000Z"""
 
-// DateTime is local time, so 1AM in UTC
->> JSON(Table({a:DateTime(2014,11,29,17,5,1,997),b:Date(2019, 4, 22),c:Time(12, 34, 56, 789)}))
-"[{""a"":""2014-11-30T01:05:01.997Z"",""b"":""2019-04-22"",""c"":""12:34:56.789""}]"
-
 >> JSON(Table({a:GUID("01234567-89AB-CDEF-0123-456789ABCDEF"),b:RGBA(18, 52, 86, 0.5),c:"https://www.microsoft.com",d:Sqrt(9)}))
 "[{""a"":""01234567-89ab-cdef-0123-456789abcdef"",""b"":""#12345680"",""c"":""https://www.microsoft.com"",""d"":3}]"
 
@@ -123,3 +109,25 @@ Error({Kind:ErrorKind.Div0})
 // Error records
 >> JSON(Filter([-2,-1,0,1,2], 1/Value>0))
 Error({Kind:ErrorKind.Div0})
+
+// Flattened tables
+>> JSON([1, 2, 3], JSONFormat.FlattenValueTables)
+"[1,2,3]"
+
+>> JSON({a:["one", "two"]}, JSONFormat.FlattenValueTables)
+"{""a"":[""one"",""two""]}"
+
+>> JSON([true, false, true], JSONFormat.FlattenValueTables)
+"[true,false,true]"
+
+// Only flatten single-column tables where the column name is 'Value'
+>> JSON([{a:1}, {a:2}], JSONFormat.FlattenValueTables)
+"[{""a"":1},{""a"":2}]"
+
+// No difference between blank records and blank values
+>> JSON([{Value:1},Blank(),{Value:3},{Value:Blank()},{Value:5}], JSONFormat.FlattenValueTables)
+"[1,null,3,null,5]"
+
+// Flattening nested tables
+>> JSON([[1,2,3],[4,5],[6]], JSONFormat.FlattenValueTables)
+"[[1,2,3],[4,5],[6]]"
diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt
@@ -0,0 +1,6 @@
+// (UTC-08:00) Pacific Time (US & Canada)
+#SETUP: OptionSetTestSetup
+#SETUP: EnableJsonFunctions
+
+>> JSON(OptionSet.Option2)
+"""option_2"""
diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt
@@ -0,0 +1,11 @@
+// (UTC-08:00) Pacific Time (US & Canada)
+#SETUP: TimeZoneInfo("Pacific Standard Time")
+#SETUP: EnableJsonFunctions
+
+// Midnight local time in Pacific Time is 8 AM UTC
+>> JSON(DateTimeValue("1970-01-01T00:00:00"))
+"""1970-01-01T08:00:00.000Z"""
+
+// DateTime is local time, so 1AM in UTC
+>> JSON(Table({a:DateTime(2014,11,29,17,5,1,997),b:Date(2019, 4, 22),c:Time(12, 34, 56, 789)}))
+"[{""a"":""2014-11-30T01:05:01.997Z"",""b"":""2019-04-22"",""c"":""12:34:56.789""}]"
diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt
@@ -0,0 +1,5 @@
+#SETUP: PowerFxV1CompatibilityRules
+
+// Error for unknown options in the second argument
+>> JSON({a:1,b:[1,2,3]}, "_U")
+Errors: Error 22-26: The value 'U' is not supported as an option in the JSON function.
diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt
@@ -0,0 +1,5 @@
+#SETUP: disable:PowerFxV1CompatibilityRules
+
+// Ignore unknown options in the second argument
+>> JSON({a:1,b:[1,2,3]}, "_U")
+"{""a"":1,""b"":[1,2,3]}"
diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs
@@ -181,7 +181,7 @@ public void TestSuggestVariableName(string suggestion)
         [InlineData("Patch({a:1, b:2}, {|", "a:", "b:")]
         [InlineData("ClearCollect(Table({a:1, b:2}), {|", "a:", "b:")]
         [InlineData("Remove(Table({a:1, b:2}), {|", "a:", "b:")]
-        [InlineData("Error(Ab|  Collect()", "Abs", "Color.OliveDrab", "ErrorKind.NotApplicable", "Match.Tab", "Table")]
+        [InlineData("Error(Ab|  Collect()", "Abs", "Color.OliveDrab", "ErrorKind.NotApplicable", "JSONFormat.FlattenValueTables", "Match.Tab", "Table")]
         public void TestSuggestMutationFunctions(string expression, params string[] expectedSuggestions)
         {
             var config = SuggestTests.Default;