Skip to content

Commit

Permalink
Merge branch 'main' into carlosff/json/unwrappedtables
Browse files Browse the repository at this point in the history
  • Loading branch information
CarlosFigueiraMSFT authored Mar 13, 2024
2 parents 349f54d + dac427f commit 5207c01
Show file tree
Hide file tree
Showing 13 changed files with 359 additions and 69 deletions.
3 changes: 3 additions & 0 deletions src/libraries/Microsoft.PowerFx.Core/Localization/Strings.cs
Original file line number Diff line number Diff line change
Expand Up @@ -629,6 +629,7 @@ internal static class TexlStrings
public static ErrorResourceKey ErrInvalidSchemaNeedTypeCol_Col = new ErrorResourceKey("ErrInvalidSchemaNeedTypeCol_Col");
public static ErrorResourceKey ErrInvalidSchemaNeedCol = new ErrorResourceKey("ErrInvalidSchemaNeedCol");
public static ErrorResourceKey ErrNeedRecord = new ErrorResourceKey("ErrNeedRecord");
public static ErrorResourceKey ErrNeedRecordOrTable = new ErrorResourceKey("ErrNeedRecordOrTable");
public static ErrorResourceKey ErrAutoRefreshNotAllowed = new ErrorResourceKey("ErrAutoRefreshNotAllowed");
public static ErrorResourceKey ErrIncompatibleRecord = new ErrorResourceKey("ErrIncompatibleRecord");
public static ErrorResourceKey ErrNeedRecord_Func = new ErrorResourceKey("ErrNeedRecord_Func");
Expand Down Expand Up @@ -792,5 +793,7 @@ internal static class TexlStrings
public static ErrorResourceKey ErrOnlyPartialAttribute = new ErrorResourceKey("ErrOnlyPartialAttribute");
public static ErrorResourceKey ErrOperationDoesntMatch = new ErrorResourceKey("ErrOperationDoesntMatch");
public static ErrorResourceKey ErrUnknownPartialOp = new ErrorResourceKey("ErrUnknownPartialOp");

public static ErrorResourceKey ErrTruncatedArgWarning = new ErrorResourceKey("ErrTruncatedArgWarning");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ private Formula GetPartialCombinedFormula(string name, PartialAttribute.Attribut
{
PartialAttribute.AttributeOperationKind.PartialAnd => GeneratePartialFunction("And", name, formulas),
PartialAttribute.AttributeOperationKind.PartialOr => GeneratePartialFunction("Or", name, formulas),
PartialAttribute.AttributeOperationKind.PartialTable => GeneratePartialFunction("TableConcatenate", name, formulas),
PartialAttribute.AttributeOperationKind.PartialTable => GeneratePartialFunction("Table", name, formulas),
PartialAttribute.AttributeOperationKind.PartialRecord => GeneratePartialFunction("MergeRecords", name, formulas),
_ => throw new InvalidOperationException("Unknown partial op while generating merged NF")
};
Expand Down
40 changes: 30 additions & 10 deletions src/libraries/Microsoft.PowerFx.Core/Texl/Builtins/Table.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@
// Licensed under the MIT license.

using System.Collections.Generic;
using System.Linq;
using Microsoft.PowerFx.Core.App.ErrorContainers;
using Microsoft.PowerFx.Core.Binding;
using Microsoft.PowerFx.Core.Entities;
using Microsoft.PowerFx.Core.Errors;
using Microsoft.PowerFx.Core.Functions;
using Microsoft.PowerFx.Core.Localization;
Expand All @@ -13,7 +15,7 @@

namespace Microsoft.PowerFx.Core.Texl.Builtins
{
// Table(rec, rec, ...)
// Table(rec/table, rec/table, ...)
internal class TableFunction : BuiltinFunction
{
public override bool IsSelfContained => true;
Expand Down Expand Up @@ -56,15 +58,16 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp
Contracts.Assert(returnType.IsTable);

// Ensure that all args (if any) are records with compatible schemas.
var rowType = DType.EmptyRecord;
var resultType = DType.EmptyRecord;
for (var i = 0; i < argTypes.Length; i++)
{
var argType = argTypes[i];
var argTypeRecord = argType.IsTableNonObjNull ? argType.ToRecord() : argType;
var isChildTypeAllowedInTable = !argType.IsDeferred && !argType.IsVoid;

if (!argType.IsRecord)
if (!argTypeRecord.IsRecord)
{
errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrNeedRecord);
errors.EnsureError(DocumentErrorSeverity.Severe, args[i], TexlStrings.ErrNeedRecordOrTable);
isValid = false;
}
else if (!isChildTypeAllowedInTable)
Expand All @@ -75,18 +78,19 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp
else
{
if (DType.TryUnionWithCoerce(
rowType,
argType,
resultType,
argTypeRecord,
context.Features,
coerceToLeftTypeOnly: context.Features.StronglyTypedBuiltinEnums || context.Features.PowerFxV1CompatibilityRules,
out var newType,
out bool coercionNeeded))
{
rowType = newType;
resultType = newType;

if (coercionNeeded)
{
CollectionUtils.Add(ref nodeToCoercedTypeMap, args[i], rowType);
var coerceType = argType.IsTable ? resultType.ToTable() : resultType;
CollectionUtils.Add(ref nodeToCoercedTypeMap, args[i], coerceType);
}
}
else
Expand All @@ -96,13 +100,29 @@ public override bool CheckTypes(CheckTypesContext context, TexlNode[] args, DTyp
}
}

Contracts.Assert(rowType.IsRecord);
Contracts.Assert(resultType.IsRecord);
}

returnType = rowType.ToTable();
returnType = resultType.ToTable();

return isValid;
}

public override void CheckSemantics(TexlBinding binding, TexlNode[] args, DType[] argTypes, IErrorContainer errors)
{
base.CheckSemantics(binding, args, argTypes, errors);

for (var i = 0; i < argTypes.Length; i++)
{
var ads = argTypes[i].AssociatedDataSources?.FirstOrDefault();

if (argTypes[i].IsTableNonObjNull && ads is IExternalTabularDataSource)
{
errors.EnsureError(DocumentErrorSeverity.Warning, args[i], TexlStrings.ErrTruncatedArgWarning, args[i].ToString(), Name);
continue;
}
}
}
}

internal class TableFunction_UO : BuiltinFunction
Expand Down
34 changes: 25 additions & 9 deletions src/libraries/Microsoft.PowerFx.Interpreter/Functions/Library.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2136,18 +2136,34 @@ public static IEnumerable<DValue<RecordValue>> StandardTableNodeRecords(IRContex

public static FormulaValue Table(IRContext irContext, FormulaValue[] args)
{
// Table literal
var records = Array.ConvertAll(
args,
arg => arg switch
// Table literal
var table = new List<DValue<RecordValue>>();

for (var i = 0; i < args.Length; i++)
{
switch (args[i])
{
RecordValue r => DValue<RecordValue>.Of(r),
BlankValue b => DValue<RecordValue>.Of(b),
_ => DValue<RecordValue>.Of((ErrorValue)arg),
});
case TableValue t:
table.AddRange(t.Rows);
break;
case RecordValue r:
table.Add(DValue<RecordValue>.Of(r));
break;
case BlankValue b when b.Type._type.IsTableNonObjNull:
break;
case BlankValue b:
table.Add(DValue<RecordValue>.Of(b));
break;
case ErrorValue e when e.Type._type.IsTableNonObjNull:
return e;
default:
table.Add(DValue<RecordValue>.Of((ErrorValue)args[i]));
break;
}
}

// Returning List to ensure that the returned table is mutable
return new InMemoryTableValue(irContext, records);
return new InMemoryTableValue(irContext, table);
}

public static ValueTask<FormulaValue> Blank(EvalVisitor runner, EvalVisitorContext context, IRContext irContext, FormulaValue[] args)
Expand Down
19 changes: 14 additions & 5 deletions src/strings/PowerFxResources.en-US.resx
Original file line number Diff line number Diff line change
Expand Up @@ -1962,6 +1962,10 @@
<value>Cannot use a non-record value in this context.</value>
<comment>Error Message.</comment>
</data>
<data name="ErrNeedRecordOrTable" xml:space="preserve">
<value>Only record or table values can be used in this context.</value>
<comment>Error Message. If a record or table is expected.</comment>
</data>
<data name="ErrIncompatibleRecord" xml:space="preserve">
<value>Cannot use this record. It may contain colliding fields of incompatible types.</value>
<comment>Error Message.</comment>
Expand Down Expand Up @@ -2883,15 +2887,16 @@
<value>Language code of the supplied text.</value>
</data>
<data name="AboutTable" xml:space="preserve">
<value>Creates a table from the specified records, with as many columns as there are unique record fields. For example: Table({key1: val1, key2: val2, ...}, ...)</value>
<value>Creates a table from the specified records and tables, with as many columns as there are unique record fields. For example: Table({key1: val1, key2: val2, ...}, ...)</value>
<comment>Description of 'Table' function.</comment>
</data>
<data name="TableArg1" xml:space="preserve">
<value>record</value>
<comment>function_parameter - Argument of the Table function - a record that will become a row in the resulting table.</comment>
<value>record_or_table</value>
<comment>function_parameter - Argument of the Table function - a record or a table that will be part of the the resulting table. Translate this string. Maintain as a single word (do not add spaces).</comment>
</data>
<data name="AboutTable_record" xml:space="preserve">
<value>A record that will become a row in the resulting table.</value>
<data name="AboutTable_record_or_table" xml:space="preserve">
<value>A record or a table that will be part of the the resulting table.</value>
<comment>Description of parameter to table function.</comment>
</data>
<data name="AboutShowColumns" xml:space="preserve">
<value>Returns a table with all columns removed from the 'source' table except the specified columns.</value>
Expand Down Expand Up @@ -4523,4 +4528,8 @@
A partial operator is the 2nd part of a statement `[Partial Op]` and can be one of "And", "Or", "Table" or "Record".
It's used to determine how to combine multiple expressions with the same name and operator.</comment>
</data>
<data name="ErrTruncatedArgWarning" xml:space="preserve">
<value>Delegation warning. The result of this argument '{0}' may be truncated for large data sets before being passed to the '{1}' function.</value>
<comment>Error message when an argument to non-delegable function has possible delegation and resulting rows may be truncated</comment>
</data>
</root>
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ [Partial And]
[InlineData("And", "And")]
[InlineData("Or", "Or")]
[InlineData("Record", "MergeRecords")]
[InlineData("Table", "TableConcatenate")]
[InlineData("Table", "Table")]
public void TestNFAttributeOperationsCombined(string op, string combinedFunctionName)
{
UserDefinitions.ProcessUserDefinitions(
Expand Down
124 changes: 124 additions & 0 deletions src/tests/Microsoft.PowerFx.Core.Tests/ExpressionTestCases/Table.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#SETUP: TableSyntaxDoesntWrapRecords

>> Table([{a:0, b:false, c:"Hello"}], [{a:1, b:true, c:"World"}])
Table({a:0,b:false,c:"Hello"},{a:1,b:true,c:"World"})

// Simple coercions
>> Table([{a:0}], [{a:true}], [{a:2}])
Table({a:0},{a:1},{a:2})

>> Table([{a:0, b:"hello"}], [{a:true}], [{a:2}])
Table({a:0,b:"hello"},{a:1,b:Blank()},{a:2,b:Blank()})

>> Table([{a:0}], [{b:true}], [{c:"Hello"}])
Table({a:0,b:Blank(),c:Blank()},{a:Blank(),b:true,c:Blank()},{a:Blank(),b:Blank(),c:"Hello"})

>> Table([{a:0}], [{b:true}], [{c:"Hello", d: {x: "World"}}])
Table({a:0,b:Blank(),c:Blank(),d:Blank()},{a:Blank(),b:true,c:Blank(),d:Blank()},{a:Blank(),b:Blank(),c:"Hello",d:{x:"World"}})

// Typed blank should be treated as blank table
>> CountRows(Table(If(1<0,[1,2,3],Blank())))
0

// untyped blank should be treated as blank record
>> CountRows(Table([{a:0, b:"hello"}], Blank()))
2

>> CountRows(Table(Sequence(3000)))
3000

// Mixing - record and table
>> Table({c:"Hello", d: {x: "World"}}, [{c:"PowerFx", d: {x: "Cool"}}])
Table({c:"Hello",d:{x:"World"}},{c:"PowerFx",d:{x:"Cool"}})

// Invalid input - trying to coerce guid and datetime
>> Table([{a:Date(2024,1,1)}], [{a:GUID("some-guid-value-1234")}])
Errors: Error 0-63: The function 'Table' has some invalid arguments.|Error 28-62: Incompatible type. The item you are trying to put into a table has a type that is not compatible with the table.

// Table type with incompatible schema
>> Table([1, 2], If(1<0, Table({Value:{a:2}})))
Errors: Error 0-44: The function 'Table' has some invalid arguments.|Error 14-43: Incompatible type. The item you are trying to put into a table has a type that is not compatible with the table.

// No arg
>> Table()
Table()

>> Table([])
Table()

>> Table(Table(Table()))
Table()

// Single argument with table
>> Table([{a:0, b:false, c:"Hello"}])
Table({a:0,b:false,c:"Hello"})

>> Table(Table([{a:0, b:false, c:"Hello"}]))
Table({a:0,b:false,c:"Hello"})

// Single argument with record
>> Table({a:0, b:false, c:"Hello"})
Table({a:0,b:false,c:"Hello"})

// Blank inputs
>> Table(Blank(), Blank())
Table(Blank(),Blank())

>> Table([1, 2], Blank(), [4, 5], Blank(), [7, 8])
Table({Value:1},{Value:2},Blank(),{Value:4},{Value:5},Blank(),{Value:7},{Value:8})

// Tables containing runtime errors
>> Table([1, 2, 3/0, 4], [5, Sqrt(-1), 7, 8])
Table({Value:1},{Value:2},{Value:Error({Kind:ErrorKind.Div0})},{Value:4},{Value:5},{Value:Error({Kind:ErrorKind.Numeric})},{Value:7},{Value:8})

>> Table(Filter([2,1,0,-1,-2], 1/Value>0), Filter([-2,-1,0,1,2], Log(Value)>0))
Table({Value:2},{Value:1},Error({Kind:ErrorKind.Div0}),Error({Kind:ErrorKind.Numeric}),Error({Kind:ErrorKind.Numeric}),Error({Kind:ErrorKind.Numeric}),{Value:2})

// coercion failures
>> Table([42],["everything"])
Table({Value:42},{Value:Error({Kind:ErrorKind.InvalidArgument})})

>> Table(["everything"], [42])
Table({Value:"everything"},{Value:"42"})

// Error function has type ObjNull, which can is both a record and a table; we treat it as a record
>> Table([{a:1}], Error({Kind:ErrorKind.Div0, Message:"Please don't divide by zero"}), {a:3}, [{a:4}])
Table({a:1},Error({Kind:ErrorKind.Div0}),{a:3},{a:4})

>> Table([{a:1}], 1/0, {a:3}, [{a:4}])
Errors: Error 0-35: The function 'Table' has some invalid arguments.|Error 16-17: Only record or table values can be used in this context.

// The second argument is a record, no ambiguity
>> Table([{a:1}], If(1/0<2,{a:2}), {a:3}, [{a:4}])
Table({a:1},Error({Kind:ErrorKind.Div0}),{a:3},{a:4})

// The second argument is a table. With an error table is passed in, the result is an error
>> Table([{a:1}], If(1/0<2,[{a:2}]), {a:3}, [{a:4}])
Error({Kind:ErrorKind.Div0})

>> Table([{a:1}], {a:Error({Kind:ErrorKind.Custom})}, {a:3}, [{a:4}])
Table({a:1},{a:Error({Kind:ErrorKind.Custom})},{a:3},{a:4})

// Passing nested record and nested table inside a record
>> Table({a: {b: {c: "Hello"}}}, {a: {b: {c: "World"}}, d: {f:Table([1, 2], [3, 4])}}, [{d: {f:Table([4, 5], [6, 7])}}])
Table({a:{b:{c:"Hello"}},d:Blank()},{a:{b:{c:"World"}},d:{f:Table({Value:1},{Value:2},{Value:3},{Value:4})}},{a:Blank(),d:{f:Table({Value:4},{Value:5},{Value:6},{Value:7})}})

// Mixing nested record and nested table inside a record - same column - type error
>> Table({a: {b: {c: "Hello"}}}, [{a: Table({e: 1}, {e: 2}, {e:5}, {f:[1, 2, 3, 4]})}])
Errors: Error 0-84: The function 'Table' has some invalid arguments.|Error 30-83: Incompatible type. The item you are trying to put into a table has a type that is not compatible with the table.

// Does not modify existing behavior of Inline value tables
>> [[1, 2, 3], [4, 5, 6]]
Table({Value:Table({Value:1},{Value:2},{Value:3})},{Value:Table({Value:4},{Value:5},{Value:6})})

>> [[{name:"John",age:85}], [{name:"Jane",age:79}]]
Table({Value:Table({age:85,name:"John"})},{Value:Table({age:79,name:"Jane"})})

>> [{name:"John",age:85}, If(1<0, {something:"hello"})]
Table({age:85,name:"John",something:Blank()},Blank())

>> [[4, 5, 6], If(1<0, [1, 2, 3]), If(1/0, [1, 2, 3])]
Table({Value:Table({Value:4},{Value:5},{Value:6})},{Value:Blank()},{Value:Error({Kind:ErrorKind.Div0})})

>> [{name:"John",age:85}, If(1<0, [1, 2, 3])]
Errors: Error 23-41: Incompatible type. The item you are trying to put into a table has a type that is not compatible with the table.
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
#SETUP: EnableExpressionChaining,MutationFunctionsTestSetup,TableSyntaxDoesntWrapRecords

>> Set(r3, {a: Table({b:1})});
Patch(r3.a, {b:1}, {b:2});
r3
{a:Table({b:2})}

>> Set(r3, {a: Table({b:1})});
Patch(r3.DisplayNamea, {b:1}, {b:3});
r3
{a:Table({b:3})}

>> Set(t_bs1, [{b:"Hello"}]);
Set(t_bs2, Table(t_bs1, {b: "World"}));
Patch(t_bs1, {b: "Hello"}, {b: "Hi"});
t_bs2
Table({b:"Hello"},{b:"World"})

>> Set(t_bs1, [{b:"Hello"}]);
Set(t_bs2, Table(t_bs1, {b: "World"}));
Patch(t_bs1, {b: "Hello"}, {b: "Hi"});
Set(t_bs2, Table(t_bs1, {b: "World"}));
t_bs2
Table({b:"Hi"},{b:"World"})
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#SETUP: PowerFxV1CompatibilityRules

//Untyped Blank inputs
>> Table(If(1<0,Blank()))
Table(Blank())

>> Table(If(1<0,Blank()), {a: 1}, [{a: 2}])
Table(Blank(),{a:1},{a:2})
Loading

0 comments on commit 5207c01

Please sign in to comment.