Merge branch 'main' into lucgen/qfe3

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "nuget" 
+    directory: "/" 
+    schedule:
+      interval: "weekly"

README.md

@@ -31,13 +31,13 @@ The easiest and most powerful ​system to express business logic for ​everyon
 
 | Name                                     | Released Package |
 |------------------------------------------|------------------|
-| Microsoft.PowerFx.Core                   | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Core?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Core/) |
-| Microsoft.PowerFx.Connectors             | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Connectors?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Connectors/) |
-| Microsoft.PowerFx.Json                   | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Json?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Json/) |
-| Microsoft.PowerFx.Core.Tests             | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Core.Tests?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Tests/) |
-| Microsoft.PowerFx.Interpreter            | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Interpreter?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Interpreter/) |
-| Microsoft.PowerFx.LanguageServerProtocol | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.LanguageServerProtocol?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.LanguageServerProtocol/) |
-| Microsoft.PowerFx.Transport.Attributes   | [![BotBuilder Badge](https://buildstats.info/nuget/Microsoft.PowerFx.Transport.Attributes?includePreReleases=true&dWidth=70)](https://www.nuget.org/packages/Microsoft.PowerFx.Transport.Attributes/) |
+| Microsoft.PowerFx.Core                   | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Core?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Core/) |
+| Microsoft.PowerFx.Connectors             | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Connectors?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Connectors/) |
+| Microsoft.PowerFx.Json                   | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Json?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Json/) |
+| Microsoft.PowerFx.Core.Tests             | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Core.Tests?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Core.Tests/) |
+| Microsoft.PowerFx.Interpreter            | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Interpreter?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Interpreter/) |
+| Microsoft.PowerFx.LanguageServerProtocol | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.LanguageServerProtocol?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.LanguageServerProtocol/) |
+| Microsoft.PowerFx.Transport.Attributes   | [![BotBuilder Badge](https://img.shields.io/nuget/vpre/Microsoft.PowerFx.Transport.Attributes?label=Latest&logo=nuget)](https://www.nuget.org/packages/Microsoft.PowerFx.Transport.Attributes/) |
 
 ## Daily Builds
 Daily builds of the Power Fx packages are published to Azure Artifacts. 

src/libraries/Microsoft.PowerFx.Connectors/ConnectorConstants.cs

@@ -10,6 +10,7 @@ public static class Constants
         public const string XMsDynamicProperties = "x-ms-dynamic-properties";
         public const string XMsDynamicSchema = "x-ms-dynamic-schema";
         public const string XMsDynamicValues = "x-ms-dynamic-values";
+        public const string XMsEnum = "x-ms-enum";
         public const string XMsEnumDisplayName = "x-ms-enum-display-name";
         public const string XMsEnumValues = "x-ms-enum-values";
         public const string XMsExplicitInput = "x-ms-explicit-input";

src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs

@@ -17,6 +17,7 @@
 using Microsoft.OpenApi.Readers;
 using Microsoft.OpenApi.Validations;
 using Microsoft.PowerFx.Connectors.Localization;
+using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.Errors;
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Types;
@@ -867,7 +868,7 @@ private async Task<FormulaValue> PostProcessResultAsync(FormulaValue result, Bas
                 ExpressionError newError = er is HttpExpressionError her
                     ? new HttpExpressionError(her.StatusCode) { Kind = er.Kind, Severity = er.Severity, Message = $"{DPath.Root.Append(new DName(Namespace)).ToDottedSyntax()}.{Name} failed: {er.Message}" }
                     : new ExpressionError() { Kind = er.Kind, Severity = er.Severity, Message = $"{DPath.Root.Append(new DName(Namespace)).ToDottedSyntax()}.{Name} failed: {er.Message}" };
-                result = FormulaValue.NewError(newError, ev.Type); 
+                result = FormulaValue.NewError(newError, ev.Type);
             }
 
             if (IsPageable && result is RecordValue rv)
@@ -1005,33 +1006,23 @@ internal static ConnectorType GetConnectorType(string valuePath, StringValue sv,
         }
 
         // Only called by ConnectorTable.GetSchema
-        internal static ConnectorType GetConnectorTypeAndTableCapabilities(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue sv, List<SqlRelationship> sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out ServiceCapabilities tableCapabilities)
+        // Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource)
+        internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue stringValue, List<SqlRelationship> sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo)
         {
             // There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work
             OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet };
-            ISwaggerSchema tableSchema = SwaggerSchema.New(new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(sv.Value, OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic _));
-            tableCapabilities = tableSchema.GetTableCapabilities();
-
-            JsonElement je = ExtractFromJson(sv, valuePath, out name, out displayName);
-
-            // Json version to be able to read SalesForce unique properties
-            ConnectorType connectorType = GetJsonConnectorTypeInternal(compatibility, je, sqlRelationships);
-            connectorType.Name = name;
-            IList<ReferencedEntity> referencedEntities = GetReferenceEntities(connectorName, sv);
+            ISwaggerSchema tableSchema = SwaggerSchema.New(new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(stringValue.Value, OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic _));
 
+            ServiceCapabilities serviceCapabilities = tableSchema.GetTableCapabilities();
             ConnectorPermission tablePermission = tableSchema.GetPermission();
-            bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly;
-
-            List<ConnectorType> primaryKeyParts = connectorType.Fields.Where(f => f.KeyType == ConnectorKeyType.Primary).OrderBy(f => f.KeyOrder).ToList();
-
-            if (primaryKeyParts.Count == 0)
-            {
-                // $$$ need to check what triggers RO for SQL
-                //isTableReadOnly = true;
-            }
-
-            connectorType.AddTabularDataSource(tableResolver, referencedEntities, sqlRelationships, new DName(name), datasetName, connectorType, tableCapabilities, isTableReadOnly);
-
+
+            JsonElement jsonElement = ExtractFromJson(stringValue, valuePath, out name, out displayName);
+            bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly;
+            IList<ReferencedEntity> referencedEntities = GetReferenceEntities(connectorName, stringValue);
+
+            ConnectorType connectorType = new ConnectorType(jsonElement, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly);
+            delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo;
+
             return connectorType;
         }
 

src/libraries/Microsoft.PowerFx.Connectors/Microsoft.PowerFx.Connectors.csproj

@@ -23,7 +23,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.OpenApi.Readers" Version="1.6.11" />   
-    <PackageReference Include="System.Text.Json" Version="6.0.8" />
+    <PackageReference Include="System.Text.Json" Version="6.0.10" />
     <PackageReference Include="System.Collections.Immutable" Version="6.0.0" />
     <PackageReference Include="System.Memory" Version="4.5.5" />
   </ItemGroup>

src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs

@@ -333,13 +333,22 @@ internal static bool TryGetOpenApiValue(IOpenApiAny openApiAny, FormulaType form
         internal static bool IsInternal(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.IsInternal() ?? false;
 
         internal static string GetVisibility(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.GetVisibility();
+
+        internal static string GetEnumName(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.GetEnumName();
 
         // Internal parameters are not showen to the user.
         // They can have a default value or be special cased by the infrastructure (like "connectionId").
         internal static bool IsInternal(this ISwaggerExtensions schema) => string.Equals(schema.GetVisibility(), "internal", StringComparison.OrdinalIgnoreCase);
 
         internal static string GetVisibility(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsVisibility, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null;
 
+        internal static string GetEnumName(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsEnum, out IOpenApiExtension openApiExt) && 
+                                                                              openApiExt is SwaggerJsonObject jsonObject &&
+                                                                              jsonObject.TryGetValue("name", out IOpenApiAny enumName) &&
+                                                                              enumName is OpenApiString enumNameStr 
+                                                                            ? enumNameStr.Value 
+                                                                            : null;
+
         internal static string GetMediaKind(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsMediaKind, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null;
 
         internal static (bool IsPresent, string Value) GetString(this IDictionary<string, IOpenApiAny> apiObj, string str) => apiObj.TryGetValue(str, out IOpenApiAny openApiAny) && openApiAny is OpenApiString openApiStr ? (true, openApiStr.Value) : (false, null);
@@ -438,7 +447,8 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                     {
                         if (schema.Enum.All(e => e is OpenApiString))
                         {
-                            OptionSet optionSet = new OptionSet("enum", schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary());
+                            string enumName = schema.GetEnumName() ?? "enum";
+                            OptionSet optionSet = new OptionSet(enumName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary());
                             return new ConnectorType(schema, openApiParameter, optionSet.FormulaType);
                         }
                         else
@@ -466,14 +476,19 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                         case null:
                         case "decimal":
                         case "currency":
-                            return new ConnectorType(schema, openApiParameter, FormulaType.Decimal);
+                            return new ConnectorType(schema, openApiParameter, FormulaType.Decimal);                       
 
                         default:
                             return new ConnectorType(error: $"Unsupported type of number: {schema.Format}");
                     }
 
+                // For testing only
+                case "fxnumber":
+                    return new ConnectorType(schema, openApiParameter, FormulaType.Number);
+
                 // Always a boolean (Format not used)
-                case "boolean": return new ConnectorType(schema, openApiParameter, FormulaType.Boolean);
+                case "boolean": 
+                    return new ConnectorType(schema, openApiParameter, FormulaType.Boolean);
 
                 // OpenAPI spec: Format could be <null>, int32, int64
                 case "integer":

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpDelegationInfo.cs

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+using System;
+
+namespace Microsoft.PowerFx.Core.Entities
+{
+    // Used by ServiceCapabilities.ToDelegationInfo for managing CDP x-ms-capabilities
+    internal class CdpDelegationInfo : TableDelegationInfo
+    {
+        public override ColumnCapabilitiesDefinition GetColumnCapability(string fieldName)
+        {
+            if (ColumnsCapabilities.TryGetValue(fieldName, out ColumnCapabilitiesBase columnCapabilitiesBase))
+            {               
+                return columnCapabilitiesBase switch
+                {
+                    ColumnCapabilities columnCapabilities => columnCapabilities.Definition,
+                    _ => throw new NotImplementedException($"{columnCapabilitiesBase.GetType().Name} not supported yet")
+                };
+            }
+
+            return null;
+        }
+    }
+}