From d8d3e8c5dd166fd8bdffb4162a1ccb8600a8183a Mon Sep 17 00:00:00 2001 From: Carlos Figueira Date: Thu, 14 Mar 2024 11:25:35 -0700 Subject: [PATCH] New option to the JSON function to serialize unwrapped arrays (#2231) Common request: ability of the JSON function to serialize [1,2,3] as `[1,2,3]` instead of `[{"Value":1},{"Value":2},{"Value":3}]` --- .../Localization/Strings.cs | 1 + .../Texl/Builtins/Json.cs | 38 ++++++++++++++++--- .../Types/Enums/EnumStoreBuilder.cs | 2 +- .../Functions/JsonFunctionImpl.cs | 31 +++++++++++++-- src/strings/PowerFxResources.en-US.resx | 4 ++ .../ExpressionTestCases/JSON.txt | 38 +++++++++++-------- .../ExpressionTestCases/JSON_OptionSets.txt | 6 +++ .../ExpressionTestCases/JSON_US.txt | 11 ++++++ .../ExpressionTestCases/JSON_V1Compat.txt | 5 +++ .../JSON_V1CompatDisabled.txt | 5 +++ .../InterpreterSuggestTests.cs | 2 +- 11 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt create mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt create mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt create mode 100644 src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs index 229bb1f09b..4a9be289fa 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs @@ -778,6 +778,7 @@ internal static class TexlStrings public static ErrorResourceKey ErrJSONArg1UnsupportedType = new ErrorResourceKey("ErrJSONArg1UnsupportedType"); public static ErrorResourceKey ErrJSONArg1ContainsUnsupportedMedia = new ErrorResourceKey("ErrJSONArg1ContainsUnsupportedMedia"); public static ErrorResourceKey ErrJSONArg2IncompatibleOptions = new ErrorResourceKey("ErrJSONArg2IncompatibleOptions"); + public static ErrorResourceKey ErrJSONArg2UnsupportedOption = new ErrorResourceKey("ErrJSONArg2UnsupportedOption"); public static ErrorResourceKey ErrJSONArg1UnsupportedNestedType = new ErrorResourceKey("ErrJSONArg1UnsupportedNestedType"); public static ErrorResourceKey ErrJSONArg1UnsupportedTypeWithNonBehavioral = new ErrorResourceKey("ErrJSONArg1UnsupportedTypeWithNonBehavioral"); public static ErrorResourceKey ErrTraceInvalidCustomRecordType = new ErrorResourceKey("ErrTraceInvalidCustomRecordType"); diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs index 0421db63ca..930d2067b1 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Json.cs @@ -17,9 +17,11 @@ namespace Microsoft.PowerFx.Core.Texl.Builtins // JSON(data:any, [format:s]) internal class JsonFunction : BuiltinFunction { - private const string _includeBinaryDataEnumValue = "B"; - private const string _ignoreBinaryDataEnumValue = "G"; - private const string _ignoreUnsupportedTypesEnumValue = "I"; + private const char _includeBinaryDataEnumValue = 'B'; + private const char _ignoreBinaryDataEnumValue = 'G'; + private const char _ignoreUnsupportedTypesEnumValue = 'I'; + private const char _flattenTableValuesEnumValue = '_'; + private const char _indentFourEnumValue = '4'; protected bool supportsLazyTypes = false; @@ -120,9 +122,33 @@ public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[ if (nodeValue != null) { - ignoreUnsupportedTypes = nodeValue.Contains(_ignoreUnsupportedTypesEnumValue); - includeBinaryData = nodeValue.Contains(_includeBinaryDataEnumValue); - ignoreBinaryData = nodeValue.Contains(_ignoreBinaryDataEnumValue); + foreach (var option in nodeValue) + { + switch (option) + { + case _ignoreBinaryDataEnumValue: + ignoreBinaryData = true; + break; + case _ignoreUnsupportedTypesEnumValue: + ignoreUnsupportedTypes = true; + break; + case _includeBinaryDataEnumValue: + includeBinaryData = true; + break; + case _flattenTableValuesEnumValue: + case _indentFourEnumValue: + // Runtime-only options + break; + default: + if (binding.Features.PowerFxV1CompatibilityRules) + { + errors.EnsureError(optionsNode, TexlStrings.ErrJSONArg2UnsupportedOption, option); + return; + } + + break; + } + } if (includeBinaryData && ignoreBinaryData) { diff --git a/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs b/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs index f32cb62d49..3686347935 100644 --- a/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs +++ b/src/libraries/Microsoft.PowerFx.Core/Types/Enums/EnumStoreBuilder.cs @@ -58,7 +58,7 @@ internal sealed class EnumStoreBuilder }, { LanguageConstants.JSONFormatEnumString, - "%s[Compact:\"\", IndentFour:\"4\", IgnoreBinaryData:\"G\", IncludeBinaryData:\"B\", IgnoreUnsupportedTypes:\"I\"]" + "%s[Compact:\"\", IndentFour:\"4\", IgnoreBinaryData:\"G\", IncludeBinaryData:\"B\", IgnoreUnsupportedTypes:\"I\", FlattenValueTables:\"_\"]" }, { LanguageConstants.TraceSeverityEnumString, diff --git a/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs b/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs index 5ed02ed9c9..b50b6cbd73 100644 --- a/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs +++ b/src/libraries/Microsoft.PowerFx.Json/Functions/JsonFunctionImpl.cs @@ -56,7 +56,7 @@ internal FormulaValue Process() using MemoryStream memoryStream = new MemoryStream(); using Utf8JsonWriter writer = new Utf8JsonWriter(memoryStream, new JsonWriterOptions() { Indented = flags.IndentFour, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }); - Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo); + Utf8JsonWriterVisitor jsonWriterVisitor = new Utf8JsonWriterVisitor(writer, _timeZoneInfo, flattenValueTables: flags.FlattenValueTables); _arguments[0].Visit(jsonWriterVisitor); writer.Flush(); @@ -89,6 +89,7 @@ private JsonFlags GetFlags() flags.IgnoreUnsupportedTypes = arg1string.Value.Contains("I"); flags.IncludeBinaryData = arg1string.Value.Contains("B"); flags.IndentFour = arg1string.Value.Contains("4"); + flags.FlattenValueTables = arg1string.Value.Contains("_"); } if (_arguments.Length > 1 && _arguments[1] is OptionSetValue arg1optionset) @@ -97,6 +98,7 @@ private JsonFlags GetFlags() flags.IgnoreUnsupportedTypes = arg1optionset.Option == "IgnoreUnsupportedTypes"; flags.IncludeBinaryData = arg1optionset.Option == "IncludeBinaryData"; flags.IndentFour = arg1optionset.Option == "IndentFour"; + flags.FlattenValueTables = arg1optionset.Option == "FlattenValueTables"; } if ((flags.IncludeBinaryData && flags.IgnoreBinaryData) || @@ -113,13 +115,15 @@ private class Utf8JsonWriterVisitor : IValueVisitor { private readonly Utf8JsonWriter _writer; private readonly TimeZoneInfo _timeZoneInfo; + private readonly bool _flattenValueTables; internal readonly List ErrorValues = new List(); - internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo) + internal Utf8JsonWriterVisitor(Utf8JsonWriter writer, TimeZoneInfo timeZoneInfo, bool flattenValueTables) { _writer = writer; _timeZoneInfo = timeZoneInfo; + _flattenValueTables = flattenValueTables; } public void Visit(BlankValue blankValue) @@ -254,6 +258,17 @@ public void Visit(TableValue tableValue) { _writer.WriteStartArray(); + var isSingleColumnValueTable = false; + if (_flattenValueTables) + { + var fieldTypes = tableValue.Type.GetFieldTypes(); + var firstField = fieldTypes.FirstOrDefault(); + if (firstField != null && !fieldTypes.Skip(1).Any() && firstField.Name.Value == TexlFunction.ColumnName_ValueStr) + { + isSingleColumnValueTable = true; + } + } + foreach (DValue row in tableValue.Rows) { if (row.IsBlank) @@ -266,7 +281,15 @@ public void Visit(TableValue tableValue) } else { - row.Value.Visit(this); + if (isSingleColumnValueTable) + { + var namedValue = row.Value.Fields.First(); + namedValue.Value.Visit(this); + } + else + { + row.Value.Visit(this); + } } } @@ -311,6 +334,8 @@ private class JsonFlags internal bool IncludeBinaryData = false; internal bool IndentFour = false; + + internal bool FlattenValueTables = false; } private static DateTime ConvertToUTC(DateTime dateTime, TimeZoneInfo fromTimeZone) diff --git a/src/strings/PowerFxResources.en-US.resx b/src/strings/PowerFxResources.en-US.resx index c6da42a26f..e88fdd6538 100644 --- a/src/strings/PowerFxResources.en-US.resx +++ b/src/strings/PowerFxResources.en-US.resx @@ -4422,6 +4422,10 @@ The JSONFormat values '{0}' and '{1}' cannot be used together. {Locked=JSONFormat} Error message shown to the user if they try to use incompatible values from the JSONFormat enumeration. The parameters are string values, members of that enumeration. For example: The JSONFormat values 'IgnoreBinaryData' and 'IncludeBinaryData' cannot be used together. + + The value '{0}' is not supported as an option in the JSON function. + {Locked=JSON} Error message shown to the user if they try to pass an option that is not supported to the JSON function. The parameter is a single-character string value. For example: The value '$' is not supported as an option in the JSON function. + The value passed to the JSON function contains media, and it is not supported by default. To allow JSON serialization of media values, make sure to use the IncludeBinaryData option in the 'format' parameter. {Locked=JSON}{Locked=IncludeBinaryData}. Error message shown to the user if they try to serialize an object that contains media values, without specifying the flag that would allow that operation to happen. Media values are any form of media (audio, images, video) that are included as part of a record or a table. The term 'format' should be translated like the term 'JSONArg2' in this same file. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt index 2e8015fe1b..e3c8060155 100644 --- a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON.txt @@ -1,7 +1,4 @@ -// (UTC-08:00) Pacific Time (US & Canada) -#SETUP: TimeZoneInfo("Pacific Standard Time") -#SETUP: OptionSetTestSetup -#SETUP: EnableJsonFunctions +#SETUP: EnableJsonFunctions >> JSON() Errors: Error 0-6: Invalid number of arguments: received 0, expected 1-2. @@ -53,9 +50,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'. >> JSON(Color.Turquoise) """#40e0d0ff""" ->> JSON(OptionSet.Option2) -"""option_2""" - >> JSON(TimeUnit.Hours) """hours""" @@ -65,10 +59,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'. >> JSON(DateTimeValue("1970-01-01T00:00:00Z")) """1970-01-01T00:00:00.000Z""" -// Midnight local time in Pacific Time is 8 AM UTC ->> JSON(DateTimeValue("1970-01-01T00:00:00")) -"""1970-01-01T08:00:00.000Z""" - // Independent from local timezone >> With({dt: DateTime(1987,6,5,4,30,0)}, JSON(DateAdd(dt,-TimeZoneOffset(dt),TimeUnit.Minutes), JSONFormat.IndentFour)) """1987-06-05T04:30:00.000Z""" @@ -94,10 +84,6 @@ Errors: Error 5-25: The JSON function cannot serialize objects of type 'Void'. >> JSON(DateTimeValue("2022-08-07T12:34:56Z")) """2022-08-07T12:34:56.000Z""" -// DateTime is local time, so 1AM in UTC ->> JSON(Table({a:DateTime(2014,11,29,17,5,1,997),b:Date(2019, 4, 22),c:Time(12, 34, 56, 789)})) -"[{""a"":""2014-11-30T01:05:01.997Z"",""b"":""2019-04-22"",""c"":""12:34:56.789""}]" - >> JSON(Table({a:GUID("01234567-89AB-CDEF-0123-456789ABCDEF"),b:RGBA(18, 52, 86, 0.5),c:"https://www.microsoft.com",d:Sqrt(9)})) "[{""a"":""01234567-89ab-cdef-0123-456789abcdef"",""b"":""#12345680"",""c"":""https://www.microsoft.com"",""d"":3}]" @@ -123,3 +109,25 @@ Error({Kind:ErrorKind.Div0}) // Error records >> JSON(Filter([-2,-1,0,1,2], 1/Value>0)) Error({Kind:ErrorKind.Div0}) + +// Flattened tables +>> JSON([1, 2, 3], JSONFormat.FlattenValueTables) +"[1,2,3]" + +>> JSON({a:["one", "two"]}, JSONFormat.FlattenValueTables) +"{""a"":[""one"",""two""]}" + +>> JSON([true, false, true], JSONFormat.FlattenValueTables) +"[true,false,true]" + +// Only flatten single-column tables where the column name is 'Value' +>> JSON([{a:1}, {a:2}], JSONFormat.FlattenValueTables) +"[{""a"":1},{""a"":2}]" + +// No difference between blank records and blank values +>> JSON([{Value:1},Blank(),{Value:3},{Value:Blank()},{Value:5}], JSONFormat.FlattenValueTables) +"[1,null,3,null,5]" + +// Flattening nested tables +>> JSON([[1,2,3],[4,5],[6]], JSONFormat.FlattenValueTables) +"[[1,2,3],[4,5],[6]]" diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt new file mode 100644 index 0000000000..fc20982262 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_OptionSets.txt @@ -0,0 +1,6 @@ +// (UTC-08:00) Pacific Time (US & Canada) +#SETUP: OptionSetTestSetup +#SETUP: EnableJsonFunctions + +>> JSON(OptionSet.Option2) +"""option_2""" diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt new file mode 100644 index 0000000000..e37514c6b3 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_US.txt @@ -0,0 +1,11 @@ +// (UTC-08:00) Pacific Time (US & Canada) +#SETUP: TimeZoneInfo("Pacific Standard Time") +#SETUP: EnableJsonFunctions + +// Midnight local time in Pacific Time is 8 AM UTC +>> JSON(DateTimeValue("1970-01-01T00:00:00")) +"""1970-01-01T08:00:00.000Z""" + +// DateTime is local time, so 1AM in UTC +>> JSON(Table({a:DateTime(2014,11,29,17,5,1,997),b:Date(2019, 4, 22),c:Time(12, 34, 56, 789)})) +"[{""a"":""2014-11-30T01:05:01.997Z"",""b"":""2019-04-22"",""c"":""12:34:56.789""}]" diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt new file mode 100644 index 0000000000..8691a54cc5 --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1Compat.txt @@ -0,0 +1,5 @@ +#SETUP: PowerFxV1CompatibilityRules + +// Error for unknown options in the second argument +>> JSON({a:1,b:[1,2,3]}, "_U") +Errors: Error 22-26: The value 'U' is not supported as an option in the JSON function. diff --git a/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt new file mode 100644 index 0000000000..38f9bad8ac --- /dev/null +++ b/src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/JSON_V1CompatDisabled.txt @@ -0,0 +1,5 @@ +#SETUP: disable:PowerFxV1CompatibilityRules + +// Ignore unknown options in the second argument +>> JSON({a:1,b:[1,2,3]}, "_U") +"{""a"":1,""b"":[1,2,3]}" diff --git a/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs b/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs index 91024a177a..b5eb07c4f7 100644 --- a/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs +++ b/src/tests/Microsoft.PowerFx.Interpreter.Tests/InterpreterSuggestTests.cs @@ -181,7 +181,7 @@ public void TestSuggestVariableName(string suggestion) [InlineData("Patch({a:1, b:2}, {|", "a:", "b:")] [InlineData("ClearCollect(Table({a:1, b:2}), {|", "a:", "b:")] [InlineData("Remove(Table({a:1, b:2}), {|", "a:", "b:")] - [InlineData("Error(Ab| Collect()", "Abs", "Color.OliveDrab", "ErrorKind.NotApplicable", "Match.Tab", "Table")] + [InlineData("Error(Ab| Collect()", "Abs", "Color.OliveDrab", "ErrorKind.NotApplicable", "JSONFormat.FlattenValueTables", "Match.Tab", "Table")] public void TestSuggestMutationFunctions(string expression, params string[] expectedSuggestions) { var config = SuggestTests.Default;