Skip to content

Commit

Permalink
Tabular connectors: Add support for SalesForce and clean code (#2319)
Browse files Browse the repository at this point in the history
  • Loading branch information
@LucGenetier
LucGenetier authored Apr 15, 2024
1 parent fad24ba commit 77d2814
Show file tree
Hide file tree
Showing 5 changed files with 3,754 additions and 149 deletions.
6 changes: 3 additions & 3 deletions src/libraries/Microsoft.PowerFx.Connectors/Environment/CdpTabularService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ public CdpTabularService(string dataset, string table)

//// TABLE METADATA SERVICE
// GET: /$metadata.json/datasets/{datasetName}/tables/{tableName}?api-version=2015-09-01
public async Task InitAsync(HttpClient httpClient, string uriPrefix, bool useV2, CancellationToken cancellationToken, ConnectorLogger logger = null)
public virtual async Task InitAsync(HttpClient httpClient, string uriPrefix, bool useV2, CancellationToken cancellationToken, ConnectorLogger logger = null)
{
cancellationToken.ThrowIfCancellationRequested();

Expand All @@ -47,7 +47,7 @@ public async Task InitAsync(HttpClient httpClient, string uriPrefix, bool useV2,
_v2 = useV2;
_uriPrefix = uriPrefix;

string uri = (_uriPrefix ?? string.Empty) + (_v2 ? "/v2" : string.Empty) + $"/$metadata.json/datasets/{DoubleEncode(DataSetName)}/tables/{DoubleEncode(TableName)}?api-version=2015-09-01";
string uri = (_uriPrefix ?? string.Empty) + (_v2 ? "/v2" : string.Empty) + $"/$metadata.json/datasets/{DoubleEncode(DataSetName, "dataset")}/tables/{DoubleEncode(TableName, "table")}?api-version=2015-09-01";

using HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, uri);
using HttpResponseMessage response = await httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
Expand All @@ -70,7 +70,7 @@ public async Task InitAsync(HttpClient httpClient, string uriPrefix, bool useV2,
}
}

private string DoubleEncode(string param)
protected virtual string DoubleEncode(string param, string paramName)
{
// we force double encoding here (in swagger, we have "x-ms-url-encoding": "double")
return HttpUtility.UrlEncode(HttpUtility.UrlEncode(param));
Expand Down
186 changes: 127 additions & 59 deletions src/libraries/Microsoft.PowerFx.Connectors/Environment/SwaggerTabularService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,41 +24,100 @@ public sealed class SwaggerTabularService : CdpTabularService
private readonly string _connectionId;
private IReadOnlyList<ConnectorFunction> _tabularFunctions;
private ConnectorFunction _metadataService;
private ConnectorFunction _createItems;
private ConnectorFunction _getItems;
private ConnectorFunction _updateItem;
private ConnectorFunction _deleteItem;
private readonly IReadOnlyDictionary<string, FormulaValue> _globalValues;
private readonly bool _defaultDataset = false;

public SwaggerTabularService(IReadOnlyDictionary<string, FormulaValue> globalValues)
: base(GetDataSetName(globalValues), GetTableName(globalValues))
: base(GetDataSetName(globalValues, out bool useDefaultDataset), GetTableName(globalValues))
{
_globalValues = globalValues;
_connectionId = TryGetString("connectionId", globalValues, out string connectorId) ? connectorId : throw new InvalidOperationException("Cannot determine connectionId.");
_defaultDataset = useDefaultDataset;
_connectionId = TryGetString("connectionId", globalValues, out string connectorId, out _) ? connectorId : throw new InvalidOperationException("Cannot determine connectionId.");
}

public static bool IsTabular(IReadOnlyDictionary<string, FormulaValue> globalValues, OpenApiDocument openApiDocument, out string error)
{
try
{
error = null;
return new SwaggerTabularService(globalValues).LoadSwaggerAndIdentifyKeyMethods(new PowerFxConfig(), openApiDocument);
}
catch (Exception ex)
{
error = ex.Message;
return false;
}
}

[Obsolete("Only used for testing")]
internal static (ConnectorFunction s, ConnectorFunction c, ConnectorFunction r, ConnectorFunction u, ConnectorFunction d) GetFunctions(IReadOnlyDictionary<string, FormulaValue> globalValues, OpenApiDocument openApiDocument)
{
return new SwaggerTabularService(globalValues).GetFunctions(new PowerFxConfig(), openApiDocument);
}

/* Schema [GET] */

private const string MetadataServiceRegex = @"/\$metadata\.json/datasets/({[^{}]+}(,{[^{}]+})?|default)/tables/{[^{}]+}$";

/* Create [POST], Read [GET] */

private const string CreateOrGetItemsRegex = @"/datasets/({[^{}]+}(,{[^{}]+})?|default)/tables/{[^{}]+}/items$";

/* Read [GET], Update [PATCH], Delete [DELETE] */

private const string GetUpdateOrDeleteItemRegex = @"/datasets/({[^{}]+}(,{[^{}]+})?|default)/tables/{[^{}]+}/items/{[^{}]+}$";

/* Version (like V2) */

private const string NameVersionRegex = @"V(?<n>[0-9]{0,2})$";

//// TABLE METADATA SERVICE
// GET: /$metadata.json/datasets/{datasetName}/tables/{tableName}?api-version=2015-09-01
internal ConnectorFunction MetadataService => _metadataService ??= GetMetadataService();
internal ConnectorFunction MetadataService => _metadataService ??= GetFunction(HttpMethod.Get, MetadataServiceRegex, 0, "metadata service");

// TABLE DATA SERVICE - CREATE
// POST: /datasets/{datasetName}/tables/{tableName}/items?api-version=2015-09-01
internal ConnectorFunction CreateItems => _createItems ??= GetFunction(HttpMethod.Post, CreateOrGetItemsRegex, 1, "Create Items");

// TABLE DATA SERVICE - READ
// GET AN ITEM - GET: /datasets/{datasetName}/tables/{tableName}/items/{id}?api-version=2015-09-01
// LIST ITEMS - GET: /datasets/{datasetName}/tables/{tableName}/items?$filter=’CreatedBy’ eq ‘john.doe’&$top=50&$orderby=’Priority’ asc, ’CreationDate’ desc
internal ConnectorFunction GetItems => _getItems ??= GetFunction(HttpMethod.Get, CreateOrGetItemsRegex, 0, "Get Items");

// TABLE DATA SERVICE - UPDATE
// PATCH: /datasets/{datasetName}/tables/{tableName}/items/{id}
internal ConnectorFunction UpdateItem => _updateItem ??= GetFunction(new HttpMethod("PATCH"), GetUpdateOrDeleteItemRegex, 2, "Update Item");

// TABLE DATA SERVICE - DELETE
// DELETE: /datasets/{datasetName}/tables/{tableName}/items/{id}
internal ConnectorFunction DeleteItem => _deleteItem ??= GetFunction(new HttpMethod("DELETE"), GetUpdateOrDeleteItemRegex, 1, "Delete Item");

public override Task InitAsync(HttpClient httpClient, string uriPrefix, bool useV2, CancellationToken cancellationToken, ConnectorLogger logger = null)
{
throw new PowerFxConnectorException("Use InitAsync with OpenApiDocument");
}

public async Task InitAsync(PowerFxConfig config, OpenApiDocument openApiDocument, HttpClient httpClient, CancellationToken cancellationToken, ConnectorLogger logger = null)
{
cancellationToken.ThrowIfCancellationRequested();

try
{
logger?.LogInformation($"Entering in {nameof(CdpTabularService)} {nameof(InitAsync)} for {DataSetName}, {TableName}");
logger?.LogInformation($"Entering in {nameof(SwaggerTabularService)} {nameof(InitAsync)} for {DataSetName}, {TableName}");

ConnectorSettings connectorSettings = new ConnectorSettings(Namespace)
if (!LoadSwaggerAndIdentifyKeyMethods(config, openApiDocument, logger))
{
IncludeInternalFunctions = true,
Compatibility = ConnectorCompatibility.SwaggerCompatibility
};

// Swagger based tabular connectors
_tabularFunctions = config.AddActionConnector(connectorSettings, openApiDocument, _globalValues, logger);
throw new PowerFxConnectorException("Cannot identify tabular methods.");
}

BaseRuntimeConnectorContext runtimeConnectorContext = new RawRuntimeConnectorContext(httpClient);
FormulaValue schema = await MetadataService.InvokeAsync(Array.Empty<FormulaValue>(), runtimeConnectorContext, cancellationToken).ConfigureAwait(false);

logger?.LogInformation($"Exiting {nameof(CdpTabularService)} {nameof(InitAsync)} for {DataSetName}, {TableName} {(schema is ErrorValue ev ? string.Join(", ", ev.Errors.Select(er => er.Message)) : string.Empty)}");
logger?.LogInformation($"Exiting {nameof(SwaggerTabularService)} {nameof(InitAsync)} for {DataSetName}, {TableName} {(schema is ErrorValue ev ? string.Join(", ", ev.Errors.Select(er => er.Message)) : string.Empty)}");

if (schema is StringValue str)
{
Expand All @@ -72,14 +131,25 @@ public async Task InitAsync(PowerFxConfig config, OpenApiDocument openApiDocumen
}
}

// TABLE DATA SERVICE - CREATE
// POST: /datasets/{datasetName}/tables/{tableName}/items?api-version=2015-09-01
internal bool LoadSwaggerAndIdentifyKeyMethods(PowerFxConfig config, OpenApiDocument openApiDocument, ConnectorLogger logger = null)
{
var (s, c, r, u, d) = GetFunctions(config, openApiDocument, logger);
return s != null && c != null && r != null && u != null && d != null;
}

// TABLE DATA SERVICE - READ
// GET AN ITEM - GET: /datasets/{datasetName}/tables/{tableName}/items/{id}?api-version=2015-09-01
internal (ConnectorFunction s, ConnectorFunction c, ConnectorFunction r, ConnectorFunction u, ConnectorFunction d) GetFunctions(PowerFxConfig config, OpenApiDocument openApiDocument, ConnectorLogger logger = null)
{
ConnectorSettings connectorSettings = new ConnectorSettings(Namespace)
{
IncludeInternalFunctions = true,
Compatibility = ConnectorCompatibility.SwaggerCompatibility
};

// LIST ITEMS - GET: /datasets/{datasetName}/tables/{tableName}/items?$filter=’CreatedBy’ eq ‘john.doe’&$top=50&$orderby=’Priority’ asc, ’CreationDate’ desc
internal ConnectorFunction GetItems => _getItems ??= GetItemsFunction();
// Swagger based tabular connectors
_tabularFunctions = config.AddActionConnector(connectorSettings, openApiDocument, _globalValues, logger);

return (MetadataService, CreateItems, GetItems, UpdateItem, DeleteItem);
}

protected override async Task<ICollection<DValue<RecordValue>>> GetItemsInternalAsync(IServiceProvider serviceProvider, ODataParameters oDataParameters, CancellationToken cancellationToken)
{
Expand All @@ -90,17 +160,16 @@ protected override async Task<ICollection<DValue<RecordValue>>> GetItemsInternal
{
executionLogger?.LogInformation($"Entering in {nameof(SwaggerTabularService)} {nameof(GetItemsAsync)} for {DataSetName}, {TableName}");

BaseRuntimeConnectorContext runtimeConnectorContext = serviceProvider.GetService<BaseRuntimeConnectorContext>() ?? throw new InvalidOperationException("Cannot determine runtime connector context.");
ODataParameters odataParameters = serviceProvider.GetService<ODataParameters>();
IReadOnlyList<NamedValue> optionalParameters = odataParameters != null ? odataParameters.GetNamedValues() : Array.Empty<NamedValue>();
BaseRuntimeConnectorContext runtimeConnectorContext = serviceProvider.GetService<BaseRuntimeConnectorContext>() ?? throw new InvalidOperationException("Cannot determine runtime connector context.");
IReadOnlyList<NamedValue> optionalParameters = oDataParameters != null ? oDataParameters.GetNamedValues() : Array.Empty<NamedValue>();

FormulaValue[] parameters = optionalParameters.Any() ? new FormulaValue[] { FormulaValue.NewRecordFromFields(optionalParameters.ToArray()) } : Array.Empty<FormulaValue>();

// Notice that there is no paging here, just get 1 page
// Use WithRawResults to ignore _getItems return type which is in the form of ![value:*[dynamicProperties:![]]] (ie. without the actual type)
FormulaValue rowsRaw = await GetItems.InvokeAsync(parameters, runtimeConnectorContext.WithRawResults(), CancellationToken.None).ConfigureAwait(false);

executionLogger?.LogInformation($"Exiting {nameof(CdpTabularService)} {nameof(GetItemsAsync)} for {DataSetName}, {TableName} {(rowsRaw is ErrorValue ev ? string.Join(", ", ev.Errors.Select(er => er.Message)) : string.Empty)}");
executionLogger?.LogInformation($"Exiting {nameof(SwaggerTabularService)} {nameof(GetItemsAsync)} for {DataSetName}, {TableName} {(rowsRaw is ErrorValue ev ? string.Join(", ", ev.Errors.Select(er => er.Message)) : string.Empty)}");
return rowsRaw is StringValue sv ? GetResult(sv.Value) : Array.Empty<DValue<RecordValue>>();
}
catch (Exception ex)
Expand All @@ -110,72 +179,71 @@ protected override async Task<ICollection<DValue<RecordValue>>> GetItemsInternal
}
}

// TABLE DATA SERVICE - UPDATE
// PATCH: /datasets/{datasetName}/tables/{tableName}/items/{id}
private static string GetDataSetName(IReadOnlyDictionary<string, FormulaValue> globalValues, out bool useDefaultDataset)
{
bool b1 = TryGetString("server", globalValues, out string server, out _);
bool b2 = TryGetString("database", globalValues, out string database, out _);

// TABLE DATA SERVICE - DELETE
// DELETE: /datasets/{datasetName}/tables/{tableName}/items/{id}
if (b1 && b2)
{
useDefaultDataset = false;
return $"{server},{database}";
}

private static string GetDataSetName(IReadOnlyDictionary<string, FormulaValue> globalValues) =>
TryGetString("dataset", globalValues, out string dataset)
? dataset
: TryGetString("server", globalValues, out string server) && TryGetString("database", globalValues, out string database)
? $"{server},{database}"
: throw new InvalidOperationException("Cannot determine dataset name.");
return TryGetString("dataset", globalValues, out string dataset, out useDefaultDataset, true)
? dataset
: throw new InvalidOperationException("Cannot determine dataset name.");
}

private static string GetTableName(IReadOnlyDictionary<string, FormulaValue> globalValues) =>
TryGetString("table", globalValues, out string table)
TryGetString("table", globalValues, out string table, out _)
? table
: throw new InvalidOperationException("Cannot determine table name.");

private static bool TryGetString(string name, IReadOnlyDictionary<string, FormulaValue> globalValues, out string str)
private static bool TryGetString(string name, IReadOnlyDictionary<string, FormulaValue> globalValues, out string str, out bool useDefaultDataset, bool allowDefault = false)
{
useDefaultDataset = false;

if (globalValues.TryGetValue(name, out FormulaValue fv) && fv is StringValue sv)
{
str = sv.Value;
return !string.IsNullOrEmpty(str);
}

if (allowDefault)
{
useDefaultDataset = true;
str = "default";
return true;
}

str = null;
return false;
}

private const string MetadataServiceRegex = @"/\$metadata\.json/datasets/{[^{}]+}(,{[^{}]+})?/tables/{[^{}]+}$";

private const string GetItemsRegex = @"/datasets/{[^{}]+}(,{[^{}]+})?/tables/{[^{}]+}/items$";

private const string NameVersionRegex = @"V(?<n>[0-9]{0,2})$";

private ConnectorFunction GetMetadataService()
private ConnectorFunction GetFunction(HttpMethod httpMethod, string regex, int numArgs, string log)
{
ConnectorFunction[] functions = _tabularFunctions.Where(tf => tf.RequiredParameters.Length == 0 && new Regex(MetadataServiceRegex).IsMatch(tf.OperationPath)).ToArray();

if (functions.Length == 0)
if (_tabularFunctions == null)
{
throw new PowerFxConnectorException("Cannot determine metadata service function.");
}

if (functions.Length > 1)
{
// When GetTableTabularV2, GetTableTabular exist, return highest version
return functions[functions.Select((cf, i) => (index: i, version: int.Parse("0" + new Regex(NameVersionRegex).Match(cf.Name).Groups["n"].Value, CultureInfo.InvariantCulture))).OrderByDescending(x => x.version).First().index];
throw new PowerFxConnectorException("Tabular functions are not initialized.");
}

return functions[0];
}

private ConnectorFunction GetItemsFunction()
{
ConnectorFunction[] functions = _tabularFunctions.Where(tf => tf.RequiredParameters.Length == 0 && new Regex(GetItemsRegex).IsMatch(tf.OperationPath)).ToArray();
ConnectorFunction[] functions = _tabularFunctions.Where(tf => tf.HttpMethod == httpMethod && tf.RequiredParameters.Length == numArgs && new Regex(regex).IsMatch(tf.OperationPath)).ToArray();

if (functions.Length == 0)
{
throw new PowerFxConnectorException("Cannot determine GetItems function.");
if (_defaultDataset)
{
throw new PowerFxConnectorException($"Cannot determine {log} function. 'dataset' value probably missing.");
}

throw new PowerFxConnectorException($"Cannot determine {log} function.");
}

if (functions.Length > 1)
{
throw new PowerFxConnectorException("Multiple GetItems functions found.");
// When GetTableTabularV2, GetTableTabular exist, return highest version
return functions[functions.Select((cf, i) => (index: i, version: int.Parse("0" + new Regex(NameVersionRegex).Match(cf.Name).Groups["n"].Value, CultureInfo.InvariantCulture))).OrderByDescending(x => x.version).First().index];
}

return functions[0];
Expand Down
Loading

0 comments on commit 77d2814

Please sign in to comment.