Tabular connectors - initial work

src/libraries/Microsoft.PowerFx.Connectors/ConnectorConstants.cs

@@ -1,17 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
-using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Text;
-
 namespace Microsoft.PowerFx.Connectors
 {
     public static class Constants
     {
         public const string XMsBodyName = "x-bodyName";
-        public const string XMsNotificationContent = "x-ms-notification-content";
         public const string XMsDynamicList = "x-ms-dynamic-list";
         public const string XMsDynamicProperties = "x-ms-dynamic-properties";
         public const string XMsDynamicSchema = "x-ms-dynamic-schema";
@@ -20,6 +14,7 @@ public static class Constants
         public const string XMsEnumValues = "x-ms-enum-values";
         public const string XMsExplicitInput = "x-ms-explicit-input";
         public const string XMsMediaKind = "x-ms-media-kind";
+        public const string XMsNotificationContent = "x-ms-notification-content";        
         public const string XMsPageable = "x-ms-pageable";
         public const string XMsRequireUserConfirmation = "x-ms-require-user-confirmation";
         public const string XMsSummary = "x-ms-summary";

src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs

@@ -228,15 +228,20 @@ internal ConnectorParameter[] HiddenRequiredParameters
         /// </summary>
         internal IEnumerable<OpenApiServer> Servers { get; init; }
 
+        /// <summary>
+        /// Filtered function.
+        /// </summary>
+        internal bool Filtered { get; }
+
         /// <summary>
         /// Dynamic schema extension on return type (response).
         /// </summary>
-        internal ConnectorDynamicSchema DynamicReturnSchema => EnsureConnectorFunction(ReturnParameterType?.DynamicSchema, FunctionList);
+        internal ConnectorDynamicSchema DynamicReturnSchema => EnsureConnectorFunction(ReturnParameterType?.DynamicSchema, GlobalContext.FunctionList);
 
         /// <summary>
         /// Dynamic schema extension on return type (response).
         /// </summary>
-        internal ConnectorDynamicProperty DynamicReturnProperty => EnsureConnectorFunction(ReturnParameterType?.DynamicProperty, FunctionList);
+        internal ConnectorDynamicProperty DynamicReturnProperty => EnsureConnectorFunction(ReturnParameterType?.DynamicProperty, GlobalContext.FunctionList);
 
         /// <summary>
         /// Return type when determined at runtime/by dynamic intellisense.
@@ -258,9 +263,10 @@ public ConnectorType ReturnParameterType
         private DType[] _parameterTypes;
 
         /// <summary>
-        /// List of functions in the same swagger file. Used for resolving dynamic schema/property.
+        /// Contains the list of functions in the same swagger file, used for resolving dynamic schema/property.
+        /// Also contains all global values.
         /// </summary>
-        internal IReadOnlyList<ConnectorFunction> FunctionList { get; }
+        internal ConnectorGlobalContext GlobalContext { get; }
 
         // Those parameters are protected by EnsureInitialized
         private int _arityMin;
@@ -286,18 +292,32 @@ public ConnectorType ReturnParameterType
 
         private readonly ConnectorLogger _configurationLogger = null;
 
-        internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List<ConnectorFunction> functionList, ConnectorLogger configurationLogger)
+        internal ConnectorFunction(OpenApiOperation openApiOperation, bool isSupported, string notSupportedReason, string name, string operationPath, HttpMethod httpMethod, ConnectorSettings connectorSettings, List<ConnectorFunction> functionList, ConnectorLogger configurationLogger, IReadOnlyDictionary<string, FormulaValue> globalValues)
         {
             Operation = openApiOperation;
             Name = name;
             OperationPath = operationPath;
             HttpMethod = httpMethod;
             ConnectorSettings = connectorSettings;
-            FunctionList = functionList;
+            GlobalContext = new ConnectorGlobalContext(functionList ?? throw new ArgumentNullException(nameof(functionList)), globalValues);
 
             _configurationLogger = configurationLogger;
             _isSupported = isSupported;
             _notSupportedReason = notSupportedReason ?? (isSupported ? string.Empty : throw new PowerFxConnectorException("Internal error on not supported reason"));
+
+            int nvCount = globalValues?.Count(nv => nv.Key != "connectionId") ?? 0;
+            if (nvCount > 0)
+            {
+                EnsureInitialized();
+                Filtered = _requiredParameters.Length < nvCount || !_requiredParameters.Take(nvCount).All(rp => globalValues.Keys.Contains(rp.Name));
+
+                if (!Filtered)
+                {
+                    _requiredParameters = _requiredParameters.Skip(nvCount).ToArray();
+                    _arityMin -= nvCount;
+                    _arityMax -= nvCount;
+                }
+            }
         }
 
         internal void SetUnsupported(string notSupportedReason)
@@ -906,10 +926,7 @@ private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicSchemaAsync(
                 return null;
             }
 
-            JsonElement je = ExtractFromJson(sv, cds.ValuePath);
-            OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet };
-            OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(je.ToString(), Microsoft.OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);
-            ConnectorType connectorType = new ConnectorType(schema, ConnectorSettings.Compatibility);
+            ConnectorType connectorType = GetConnectorType(cds.ValuePath, sv, ConnectorSettings.Compatibility);
 
             if (connectorType.HasErrors)
             {
@@ -923,6 +940,16 @@ private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicSchemaAsync(
             return connectorType;
         }
 
+        internal static ConnectorType GetConnectorType(string valuePath, StringValue sv, ConnectorCompatibility compatibility)
+        {
+            JsonElement je = ExtractFromJson(sv, valuePath);
+            OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet };
+            OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);
+            ConnectorType connectorType = new ConnectorType(schema, compatibility);
+
+            return connectorType;
+        }
+
         private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicPropertyAsync(NamedValue[] knownParameters, BaseRuntimeConnectorContext runtimeContext, ConnectorDynamicProperty cdp, CancellationToken cancellationToken)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -941,10 +968,7 @@ private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicPropertyAsyn
                 return null;
             }
 
-            JsonElement je = ExtractFromJson(sv, cdp.ItemValuePath);
-            OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet };
-            OpenApiSchema schema = new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(je.ToString(), OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);
-            ConnectorType connectorType = new ConnectorType(schema, ConnectorSettings.Compatibility);
+            ConnectorType connectorType = GetConnectorType(cdp.ItemValuePath, sv, ConnectorSettings.Compatibility);
 
             if (connectorType.HasErrors)
             {
@@ -1060,7 +1084,7 @@ private async Task<FormulaValue> ConnectorDynamicCallAsync(ConnectorDynamicApi d
         {
             cancellationToken.ThrowIfCancellationRequested();
 
-            EnsureConnectorFunction(dynamicApi, FunctionList);
+            EnsureConnectorFunction(dynamicApi, GlobalContext.FunctionList);
             if (dynamicApi.ConnectorFunction == null)
             {
                 runtimeContext.ExecutionLogger?.LogError($"Exiting {this.LogFunction(nameof(ConnectorDynamicCallAsync))}, {nameof(dynamicApi.ConnectorFunction)} is null");
@@ -1115,7 +1139,7 @@ private FormulaValue[] GetArguments(ConnectorDynamicApi dynamicApi, NamedValue[]
         {
             List<FormulaValue> arguments = new List<FormulaValue>();
 
-            ConnectorFunction functionToBeCalled = EnsureConnectorFunction(dynamicApi, FunctionList).ConnectorFunction;
+            ConnectorFunction functionToBeCalled = EnsureConnectorFunction(dynamicApi, GlobalContext.FunctionList).ConnectorFunction;
 
             foreach (ConnectorParameter connectorParameter in functionToBeCalled.RequiredParameters)
             {

src/libraries/Microsoft.PowerFx.Connectors/EngineExtensions.cs

@@ -0,0 +1,36 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System.Collections.Generic;
+using Microsoft.PowerFx.Core.IR;
+using Microsoft.PowerFx.Core.IR.Nodes;
+
+namespace Microsoft.PowerFx.Connectors
+{
+    public static class EngineExtensions
+    {
+        // To support tabular connectors, we need to use an IR transform to inject the ServiceProvider at runtime (which contains the HttpClient)
+        public static void EnableTabularConnectors(this Engine engine)
+        {
+            engine.IRTransformList.Add(new TabularTransform());
+        }
+
+        private class TabularTransform : IRTransform
+        {
+            public TabularTransform()
+                : base(nameof(TabularTransform))
+            {
+            }
+
+            public override IntermediateNode Transform(IntermediateNode node, ICollection<ExpressionError> errors)
+            {
+                TabularIRVisitor visitor = new TabularIRVisitor();
+                TabularIRVisitor.Context context = new TabularIRVisitor.Context();
+
+                TabularIRVisitor.RetVal ret = node.Accept(visitor, context);
+                IntermediateNode result = visitor.Materialize(ret);
+                return result;
+            }
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Connectors/Environment/PowerFxConfigExtensions.cs

@@ -4,9 +4,13 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
 using Microsoft.OpenApi.Models;
 using Microsoft.PowerFx.Connectors;
 using Microsoft.PowerFx.Core.Functions;
+using Microsoft.PowerFx.Types;
 using static Microsoft.PowerFx.Connectors.ConnectorHelperFunctions;
 
 namespace Microsoft.PowerFx
@@ -24,13 +28,14 @@ public static class ConfigExtensions
         /// <param name="connectorSettings">Connector settings containing Namespace and MaxRows to be returned.</param>        
         /// <param name="openApiDocument">An API document. This can represent multiple formats, including Swagger 2.0 and OpenAPI 3.0.</param>
         /// <param name="configurationLogger">Logger.</param>
+        /// <param name="globalValues">Global Values.</param>
         /// <returns>List of connector functions.</returns>
-        public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxConfig config, ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null)
+        public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxConfig config, ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary<string, FormulaValue> globalValues = null)
         {            
             try
             {
                 configurationLogger?.LogInformation($"Entering in ConfigExtensions.{nameof(AddActionConnector)}, with {nameof(ConnectorSettings)} {LogConnectorSettings(connectorSettings)}");
-                IReadOnlyList<ConnectorFunction> connectorFunctions = AddActionConnectorInternal(config, connectorSettings, openApiDocument, configurationLogger);
+                IReadOnlyList<ConnectorFunction> connectorFunctions = AddActionConnectorInternal(config, connectorSettings, openApiDocument, configurationLogger, globalValues);
                 configurationLogger?.LogInformation($"Exiting ConfigExtensions.{nameof(AddActionConnector)}, returning {connectorFunctions.Count()} functions");
 
                 return connectorFunctions;
@@ -42,15 +47,15 @@ public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxCo
             }
         }
 
-        internal static IReadOnlyList<ConnectorFunction> AddActionConnectorInternal(this PowerFxConfig config, ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null)
+        internal static IReadOnlyList<ConnectorFunction> AddActionConnectorInternal(this PowerFxConfig config, ConnectorSettings connectorSettings, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary<string, FormulaValue> globalValues = null)
         {
             if (config == null)
             {
                 configurationLogger?.LogError($"PowerFxConfig is null, cannot add functions");
                 return null;
             }
 
-            (List<ConnectorFunction> connectorFunctions, List<ConnectorTexlFunction> texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger);
+            (List<ConnectorFunction> connectorFunctions, List<ConnectorTexlFunction> texlFunctions) = OpenApiParser.ParseInternal(connectorSettings, openApiDocument, configurationLogger, globalValues);
             foreach (TexlFunction function in texlFunctions)
             {
                 config.AddFunction(function);
@@ -68,13 +73,14 @@ internal static IReadOnlyList<ConnectorFunction> AddActionConnectorInternal(this
         /// <param name="namespace">Namespace name.</param>
         /// <param name="openApiDocument">An API document. This can represent multiple formats, including Swagger 2.0 and OpenAPI 3.0.</param>
         /// <param name="configurationLogger">Logger.</param>
+        /// <param name="globalValues">Global Values.</param>
         /// <returns>List of connector functions.</returns>
-        public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxConfig config, string @namespace, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null)
+        public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxConfig config, string @namespace, OpenApiDocument openApiDocument, ConnectorLogger configurationLogger = null, IReadOnlyDictionary<string, FormulaValue> globalValues = null)
         {            
             try
             {
                 configurationLogger?.LogInformation($"Entering in ConfigExtensions.{nameof(AddActionConnector)}, with {nameof(ConnectorSettings)} Namespace {@namespace ?? Null(nameof(@namespace))}");
-                IReadOnlyList<ConnectorFunction> connectorFunctions = AddActionConnectorInternal(config, new ConnectorSettings(@namespace), openApiDocument, configurationLogger);
+                IReadOnlyList<ConnectorFunction> connectorFunctions = AddActionConnectorInternal(config, new ConnectorSettings(@namespace), openApiDocument, configurationLogger, globalValues);
 
                 if (connectorFunctions == null)
                 {
@@ -90,5 +96,16 @@ public static IReadOnlyList<ConnectorFunction> AddActionConnector(this PowerFxCo
                 throw;
             }            
         }
+
+        public static async Task<ConnectorTableValue> AddTabularConnector(this PowerFxConfig config, string tableName, TabularService tabularService, BaseRuntimeConnectorContext runtimeConnectorContext, CancellationToken cancellationToken, ConnectorLogger configurationLogger = null)
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            RecordType recordType = await tabularService.InitAsync(config, tableName, runtimeConnectorContext, cancellationToken).ConfigureAwait(false);
+
+            return recordType == null
+                ? throw new InvalidOperationException("Cannot determine table schema")
+                : new ConnectorTableValue(tabularService, recordType);
+        }        
     }
 }