Adding support to Unicode characters for CDP table

src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs

@@ -110,12 +110,11 @@ protected override async Task<IReadOnlyCollection<DValue<RecordValue>>> GetItems
 
             string queryParams = (odataParameters != null) ? "&" + odataParameters.ToQueryString() : string.Empty;
 
-            Uri uri = new Uri(
-                   (_uriPrefix ?? string.Empty) +
+            var uri = (_uriPrefix ?? string.Empty) +
                    (CdpTableResolver.UseV2(_uriPrefix) ? "/v2" : string.Empty) +
-                   $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams, UriKind.Relative);
+                   $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams;
 
-            string text = await GetObject(_httpClient, $"List items ({nameof(GetItemsInternalAsync)})", uri.ToString(), null, cancellationToken, executionLogger).ConfigureAwait(false);
+            string text = await GetObject(_httpClient, $"List items ({nameof(GetItemsInternalAsync)})", uri, null, cancellationToken, executionLogger).ConfigureAwait(false);
             return !string.IsNullOrWhiteSpace(text) ? GetResult(text) : Array.Empty<DValue<RecordValue>>();
         }        
 

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Microsoft.PowerFx.Connectors.Tests.Shared.projitems

@@ -49,6 +49,8 @@
     <Compile Include="$(MSBuildThisFileDirectory)Helpers\LoggingTestServer.cs" />
   </ItemGroup>
   <ItemGroup>
+    <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL Server Get First Customers_Aßþ.json" />
+    <EmbeddedResource Include="$(MSBuildThisFileDirectory)Responses\SQL Server Load Customers_Aßþ.json" />
     <None Include="$(MSBuildThisFileDirectory)Owl.png">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs

@@ -83,9 +83,9 @@ public async Task SQL_CdpTabular_GetTables()
             IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
 
             Assert.NotNull(tables);
-            Assert.Equal(4, tables.Count());
-            Assert.Equal("[dbo].[Customers],[dbo].[Orders],[dbo].[Products],[sys].[database_firewall_rules]", string.Join(",", tables.Select(t => t.TableName)));
-            Assert.Equal("Customers,Orders,Products,database_firewall_rules", string.Join(",", tables.Select(t => t.DisplayName)));
+            Assert.Equal(5, tables.Count());
+            Assert.Equal("[dbo].[Customers],[dbo].[Orders],[dbo].[Products],[sys].[database_firewall_rules],[dbo].[Customers_Aßþ]", string.Join(",", tables.Select(t => t.TableName)));
+            Assert.Equal("Customers,Orders,Products,database_firewall_rules,Customers_Aßþ", string.Join(",", tables.Select(t => t.DisplayName)));
 
             CdpTable connectorTable = tables.First(t => t.DisplayName == "Customers");
 
@@ -286,6 +286,114 @@ public async Task SQL_CdpTabular_GetTables3()
             Assert.True(connectorTable.IsInitialized);
         }
 
+        [Fact]
+        public async Task SQL_CdpTabular_GetTables4()
+        {
+            using var testConnector = new LoggingTestServer(null /* no swagger */, _output);
+            var config = new PowerFxConfig(Features.PowerFxV1);
+            var engine = new RecalcEngine(config);
+
+            ConsoleLogger logger = new ConsoleLogger(_output);
+            using var httpClient = new HttpClient(testConnector);
+            string connectionId = "c1a4e9f52ec94d55bb82f319b3e33a6a";
+            string jwt = "eyJ0eXAiOiJKV1QiL...";
+            using var client = new PowerPlatformConnectorClient("firstrelease-003.azure-apihub.net", "49970107-0806-e5a7-be5e-7c60e2750f01", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" };
+
+            testConnector.SetResponseFromFile(@"Responses\SQL GetDatasetsMetadata.json");
+            DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+
+            Assert.NotNull(dm);
+            Assert.Null(dm.Blob);
+
+            Assert.Equal("{server},{database}", dm.DatasetFormat);
+            Assert.NotNull(dm.Tabular);
+            Assert.Equal("dataset", dm.Tabular.DisplayName);
+            Assert.Equal("mru", dm.Tabular.Source);
+            Assert.Equal("Table", dm.Tabular.TableDisplayName);
+            Assert.Equal("Tables", dm.Tabular.TablePluralName);
+            Assert.Equal("single", dm.Tabular.UrlEncoding);
+            Assert.NotNull(dm.Parameters);
+            Assert.Equal(2, dm.Parameters.Count);
+
+            Assert.Equal("Server name.", dm.Parameters.First().Description);
+            Assert.Equal("server", dm.Parameters.First().Name);
+            Assert.True(dm.Parameters.First().Required);
+            Assert.Equal("string", dm.Parameters.First().Type);
+            Assert.Equal("double", dm.Parameters.First().UrlEncoding);
+            Assert.Null(dm.Parameters.First().XMsDynamicValues);
+            Assert.Equal("Server name", dm.Parameters.First().XMsSummary);
+
+            Assert.Equal("Database name.", dm.Parameters.Skip(1).First().Description);
+            Assert.Equal("database", dm.Parameters.Skip(1).First().Name);
+            Assert.True(dm.Parameters.Skip(1).First().Required);
+            Assert.Equal("string", dm.Parameters.Skip(1).First().Type);
+            Assert.Equal("double", dm.Parameters.Skip(1).First().UrlEncoding);
+            Assert.NotNull(dm.Parameters.Skip(1).First().XMsDynamicValues);
+            Assert.Equal("/v2/databases?server={server}", dm.Parameters.Skip(1).First().XMsDynamicValues.Path);
+            Assert.Equal("value", dm.Parameters.Skip(1).First().XMsDynamicValues.ValueCollection);
+            Assert.Equal("Name", dm.Parameters.Skip(1).First().XMsDynamicValues.ValuePath);
+            Assert.Equal("DisplayName", dm.Parameters.Skip(1).First().XMsDynamicValues.ValueTitle);
+            Assert.Equal("Database name", dm.Parameters.Skip(1).First().XMsSummary);
+
+            CdpDataSource cds = new CdpDataSource("pfxdev-sql.database.windows.net,connectortest");
+
+            testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables.json");
+            IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+
+            Assert.NotNull(tables);
+            Assert.Equal(5, tables.Count());
+            Assert.Equal("[dbo].[Customers],[dbo].[Orders],[dbo].[Products],[sys].[database_firewall_rules],[dbo].[Customers_Aßþ]", string.Join(",", tables.Select(t => t.TableName)));
+            Assert.Equal("Customers,Orders,Products,database_firewall_rules,Customers_Aßþ", string.Join(",", tables.Select(t => t.DisplayName)));
+
+            CdpTable connectorTable = tables.First(t => t.DisplayName == "Customers_Aßþ");
+
+            Assert.False(connectorTable.IsInitialized);
+            Assert.Equal("Customers_Aßþ", connectorTable.DisplayName);
+
+            testConnector.SetResponseFromFiles(@"Responses\SQL Server Load Customers_Aßþ.json");
+            await connectorTable.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+            Assert.True(connectorTable.IsInitialized);
+
+            CdpTableValue sqlTable = connectorTable.GetTableValue();
+            Assert.True(sqlTable._tabularService.IsInitialized);
+            Assert.True(sqlTable.IsDelegable);
+            Assert.Equal("r*[Address:s, Country:s, CustomerId:w, Name:s, Phone:s]", sqlTable.Type.ToStringWithDisplayNames());
+
+            HashSet<IExternalTabularDataSource> ads = sqlTable.Type._type.AssociatedDataSources;
+            Assert.NotNull(ads);
+            Assert.Single(ads);
+
+            DataSourceInfo dataSourceInfo = Assert.IsType<DataSourceInfo>(ads.First());
+            Assert.NotNull(dataSourceInfo);
+
+            Assert.Equal("Customers_Aßþ", dataSourceInfo.EntityName.Value);
+            Assert.True(dataSourceInfo.IsDelegatable);
+            Assert.True(dataSourceInfo.IsPageable);
+            Assert.True(dataSourceInfo.IsRefreshable);
+            Assert.True(dataSourceInfo.IsSelectable);
+            Assert.True(dataSourceInfo.IsWritable);
+
+            Assert.Equal("Customers_Aßþ", dataSourceInfo.Name);
+            Assert.True(dataSourceInfo.RequiresAsync);
+
+            Assert.Null(sqlTable.Relationships);
+
+            SymbolValues symbolValues = new SymbolValues().Add("Customers_Aßþ", sqlTable);
+            RuntimeConfig rc = new RuntimeConfig(symbolValues).AddService<ConnectorLogger>(logger);
+
+            // Expression with tabular connector
+            string expr = @"First(Customers_Aßþ).Address";
+            CheckResult check = engine.Check(expr, options: new ParserOptions() { AllowsSideEffects = true }, symbolTable: symbolValues.SymbolTable);
+            Assert.True(check.IsSuccess);
+
+            // Use tabular connector. Internally we'll call CdpTableValue.GetRowsInternal to get the data
+            testConnector.SetResponseFromFile(@"Responses\SQL Server Get First Customers_Aßþ.json");
+            FormulaValue result = await check.GetEvaluator().EvalAsync(CancellationToken.None, rc);
+
+            StringValue name = Assert.IsType<StringValue>(result);
+            Assert.Equal("Juigné", name.Value);
+        }
+
         [Fact]
         public async Task SQL_CdpTabular_JoinCapabilityTest()
         {

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SQL GetTables.json

@@ -16,6 +16,10 @@
     {
       "Name": "[sys].[database_firewall_rules]",
       "DisplayName": "sys.database_firewall_rules"
+    },
+    {
+      "Name": "[dbo].[Customers_Aßþ]",
+      "DisplayName": "Customers_Aßþ"
     }
   ]
 }

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SQL Server Get First Customers_Aßþ.json

@@ -0,0 +1,23 @@
+{
+  "@odata.context": "https://tip2-001.azure-apihub.net/apim/sql/5a6d0eb8c4dd44178c46d85caa5593dc/$metadata#datasets('pfxdev-sql.database.windows.net%2Cconnectortest')/tables('Customers')/items",
+  "value": [
+    {
+      "@odata.etag": "",
+      "ItemInternalId": "12a4c2d9-40f1-4840-8646-5955bbe6d705",
+      "CustomerId": 1,
+      "Name": "Luc",
+      "Address": "Juign\u00e9",
+      "Country": "France",
+      "Phone": "+33241010101"
+    },
+    {
+      "@odata.etag": "",
+      "ItemInternalId": "12a4c2d9-40f1-4842-8646-5955bbe6d705",
+      "CustomerId": 2,
+      "Name": "Mike",
+      "Address": "Aßþ",
+      "Country": "USA",
+      "Phone": "+1-425-705-0000"
+    }
+  ]
+}

src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/Responses/SQL Server Load Customers_Aßþ.json

@@ -0,0 +1,63 @@
+{
+  "name": "Customers_Aßþ",
+  "title": "Customers_Aßþ",
+  "x-ms-permission": "read-write",
+  "x-ms-capabilities": {
+    "sortRestrictions": { "sortable": true },
+    "filterRestrictions": { "filterable": true },
+    "selectRestrictions": { "selectable": true },
+    "filterFunctionSupport": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "now", "not", "and", "or", "day", "month", "year", "hour", "minute", "second", "date", "time", "totaloffsetminutes", "totalseconds", "round", "floor", "ceiling", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat", "sum", "min", "max", "average", "countdistinct", "null" ]
+  },
+  "schema": {
+    "type": "array",
+    "items": {
+      "type": "object",
+      "required": [ "Name" ],
+      "properties": {
+        "CustomerId": {
+          "title": "CustomerId",
+          "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "add", "sub", "mul", "div", "mod", "negate", "sum", "average" ] },
+          "type": "integer",
+          "format": "int32",
+          "minimum": -2147483648,
+          "maximum": 2147483647,
+          "x-ms-permission": "read-only",
+          "x-ms-sort": "none"
+        },
+        "Name": {
+          "title": "Name",
+          "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] },
+          "type": "string",
+          "maxLength": 50,
+          "x-ms-permission": "read-write",
+          "x-ms-sort": "none"
+        },
+        "Address": {
+          "title": "Address",
+          "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] },
+          "type": "string",
+          "maxLength": 80,
+          "x-ms-permission": "read-write",
+          "x-ms-sort": "none"
+        },
+        "Country": {
+          "title": "Country",
+          "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] },
+          "type": "string",
+          "maxLength": 40,
+          "x-ms-permission": "read-write",
+          "x-ms-sort": "none"
+        },
+        "Phone": {
+          "title": "Phone",
+          "x-ms-capabilities": { "filterFunctions": [ "lt", "le", "eq", "ne", "gt", "ge", "min", "max", "countdistinct", "contains", "startswith", "endswith", "length", "indexof", "replace", "substring", "substringof", "tolower", "toupper", "trim", "concat" ] },
+          "type": "string",
+          "maxLength": 50,
+          "x-ms-permission": "read-write",
+          "x-ms-sort": "none"
+        }
+      }
+    },
+    "x-ms-permission": "read-write"
+  }
+}