Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Culture support for in operator and Sort function #2539

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Culture support for in operator and Sort function #2539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ed1d335
Updates
gregli-msft Jul 10, 2024
e44f670
Merge branch 'main' into gregli/culture-tr
gregli-msft Aug 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions src/libraries/Microsoft.PowerFx.Core/Public/Values/StringValue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,6 @@ public override void Visit(IValueVisitor visitor)
visitor.Visit(this);
}

internal StringValue ToLower()
{
return new StringValue(IRContext.NotInSource(FormulaType.String), Value.ToLowerInvariant());
}

public override void ToExpression(StringBuilder sb, FormulaValueSerializerSettings settings)
{
sb.Append($"\"{CharacterUtils.ExcelEscapeString(Value)}\"");
Expand Down
23 changes: 13 additions & 10 deletions src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryOperators.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT license.

using System;
using System.Globalization;
using System.Linq;
using System.Text;
using Microsoft.PowerFx.Core.IR;
Expand Down Expand Up @@ -720,9 +721,9 @@ private static BooleanValue NotEqualPolymorphic(IRContext irContext, FormulaValu
}

// See in_SS in JScript membershipReplacementFunctions
public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> StringInOperator(bool exact)
{
return (irContext, args) =>
return (services, irContext, args) =>
{
var left = args[0];
var right = args[1];
Expand All @@ -738,23 +739,25 @@ public static Func<IRContext, FormulaValue[], FormulaValue> StringInOperator(boo

var leftStr = (StringValue)left;
var rightStr = (StringValue)right;

return new BooleanValue(irContext, rightStr.Value.IndexOf(leftStr.Value, exact ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase) >= 0);
return new BooleanValue(irContext, services.GetService<CultureInfo>().CompareInfo.IndexOf(rightStr.Value, leftStr.Value, exact ? CompareOptions.Ordinal : CompareOptions.IgnoreCase) >= 0);
};
}

// Left is a scalar. Right is a single-column table.
// See in_ST()
public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
public static Func<IServiceProvider, IRContext, FormulaValue[], FormulaValue> InScalarTableOperator(bool exact)
{
return (irContext, args) =>
return (services, irContext, args) =>
{
var left = args[0];
var right = args[1];

var right = args[1];

var cultureInfo = services.GetService<CultureInfo>();

if (!exact && left is StringValue strLhs)
{
left = strLhs.ToLower();
left = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strLhs.Value));
}

var source = (TableValue)right;
Expand All @@ -767,7 +770,7 @@ public static Func<IRContext, FormulaValue[], FormulaValue> InScalarTableOperato

if (!exact && rhs is StringValue strRhs)
{
rhs = strRhs.ToLower();
rhs = new StringValue(IRContext.NotInSource(FormulaType.String), cultureInfo.TextInfo.ToLower(strRhs.Value));
}

if (RuntimeHelpers.AreEqual(left, rhs))
Expand Down
28 changes: 18 additions & 10 deletions src/libraries/Microsoft.PowerFx.Interpreter/Functions/LibraryTable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -652,31 +652,31 @@ public static async ValueTask<FormulaValue> SortTable(EvalVisitor runner, EvalVi

if (allNumbers)
{
return SortValueType<NumberValue, double>(pairs, irContext, compareToResultModifier);
return SortValueType<NumberValue, double>(pairs, runner, irContext, compareToResultModifier);
}
else if (allDecimals)
{
return SortValueType<DecimalValue, decimal>(pairs, irContext, compareToResultModifier);
return SortValueType<DecimalValue, decimal>(pairs, runner, irContext, compareToResultModifier);
}
else if (allStrings)
{
return SortValueType<StringValue, string>(pairs, irContext, compareToResultModifier);
return SortValueType<StringValue, string>(pairs, runner, irContext, compareToResultModifier);
}
else if (allBooleans)
{
return SortValueType<BooleanValue, bool>(pairs, irContext, compareToResultModifier);
return SortValueType<BooleanValue, bool>(pairs, runner, irContext, compareToResultModifier);
}
else if (allDatetimes)
{
return SortValueType<DateTimeValue, DateTime>(pairs, irContext, compareToResultModifier);
return SortValueType<DateTimeValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
}
else if (allDates)
{
return SortValueType<DateValue, DateTime>(pairs, irContext, compareToResultModifier);
return SortValueType<DateValue, DateTime>(pairs, runner, irContext, compareToResultModifier);
}
else if (allTimes)
{
return SortValueType<TimeValue, TimeSpan>(pairs, irContext, compareToResultModifier);
return SortValueType<TimeValue, TimeSpan>(pairs, runner, irContext, compareToResultModifier);
}
else if (allOptionSets)
{
Expand Down Expand Up @@ -1103,7 +1103,7 @@ private static FormulaValue DistinctValueType(List<(DValue<RecordValue> row, For
return new InMemoryTableValue(irContext, result);
}

private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, IRContext irContext, int compareToResultModifier)
private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<(DValue<RecordValue> row, FormulaValue sortValue)> pairs, EvalVisitor runner, IRContext irContext, int compareToResultModifier)
where TPFxPrimitive : PrimitiveValue<TDotNetPrimitive>
where TDotNetPrimitive : IComparable<TDotNetPrimitive>
{
Expand All @@ -1119,8 +1119,16 @@ private static FormulaValue SortValueType<TPFxPrimitive, TDotNetPrimitive>(List<
}

var n1 = a.sortValue as TPFxPrimitive;
var n2 = b.sortValue as TPFxPrimitive;
return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
var n2 = b.sortValue as TPFxPrimitive;
CultureInfo culture;
if (n1.Value is string n1s && n2.Value is string n2s && (culture = runner.GetService<CultureInfo>()) != null)
{
return culture.CompareInfo.Compare(n1s, n2s) * compareToResultModifier;
}
else
{
return n1.Value.CompareTo(n2.Value) * compareToResultModifier;
}
});

return new InMemoryTableValue(irContext, pairs.Select(pair => pair.row));
Expand Down
178 changes: 178 additions & 0 deletions src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/Culture_en-US.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
#SETUP: RegEx,CultureInfo("en-US"),PowerFxV1CompatibilityRules,ConsistentOneColumnTableResult

// Four types of letter I
// Dotted Dotless
// Upper İ U+0130 I U+0049
// Lower i U+0069 ı U+0131

>> Language()
"en-US"

>> "İ" = UniChar( Hex2Dec( "0130") )
true

>> "ı" = UniChar( Hex2Dec( "0131" ) )
true

// UPPER, LOWER, PROPER

>> Upper( "i" )
"I"

>> Lower( "I" )
"i"

>> Upper( "i" ) = "I"
true

>> Lower( "I" ) = "i"
true

>> Lower( "quit" ) = Lower( "QUIT" )
true

>> Lower( "quit" ) = Lower( "QUİT" )
true

>> Lower( "quıt" ) = Lower( "QUIT" )
false

>> Upper( "quit" ) = Upper( "QUIT" )
true

>> Proper( "Iabc" )
"Iabc"

>> Proper( "iabc" )
"Iabc"

// VALUE, DECIMAL, FLOAT

>> Value( "123,456" )
123456

>> Value( "123,456", "tr-TR" )
123.456

>> Decimal( "123,456" )
123456

>> Decimal( "123,456", "tr-TR" )
123.456

>> Float( "123,456" )
123456

>> Float( "123,456", "tr-TR" )
123.456

// TEXT

>> Text( DateTime(2010,1,1,14,0,0,0), "mmm ddd yyyy AM/PM" )
"Jan Fri 2010 PM"

>> Text( DateTime(2020,1,1,2,0,0,0), "mmmm dddd yyyy AM/PM" )
"January Wednesday 2020 AM"

>> Text( 123456789, "#,###.00" )
"123,456,789.00"

>> Text( 123456789, "#.###,00" )
"123456789.00000"

// IN AND EXACTIN

>> "i" in "SIGH"
true

>> "I" in "sigh"
true

>> "i" exactin "SIGH"
false

>> "I" exactin "sigh"
false

>> "I" exactin "SIGH"
true

>> "i" exactin "sigh"
true

>> "sIGh" in ["sigh","bcde"]
true

>> "siGh" in ["SIGH","bcde"]
true

>> "sIGH" in ["sigh","bcde"]
true

>> "siGH" in ["bcde","sIgh"]
true

>> "SIgh" in ["bcde","sigh"]
true

// SORT
// Relative order of i, I, ı, İ are different between en-US and tr-TR

>> Sort( [ "Z", "İ", "z", "I", "J", "j", "ı", "a", "h", "i", "A", "H"], Value )
Table({Value:"a"},{Value:"A"},{Value:"h"},{Value:"H"},{Value:"i"},{Value:"I"},{Value:"İ"},{Value:"ı"},{Value:"j"},{Value:"J"},{Value:"z"},{Value:"Z"})

>> SortByColumns( [ "Z", "İ", "z", "I", "J", "j", "ı", "a", "h", "i", "A", "H"], "Value" )
Table({Value:"a"},{Value:"A"},{Value:"h"},{Value:"H"},{Value:"i"},{Value:"I"},{Value:"İ"},{Value:"ı"},{Value:"j"},{Value:"J"},{Value:"z"},{Value:"Z"})

>> Concat( Sort( Split( "j J k K l L m M n N o O p P r R s S t T u U v V y Y z Z Ç ç Ş ş Ü ü Ö ö İ ı Ğ ğ a A b B c C d D e E f F g G h H i I", " " ), Value ), Value, " " )
"a A b B c C ç Ç d D e E f F g G ğ Ğ h H i I İ ı j J k K l L m M n N o O ö Ö p P r R s S ş Ş t T u U ü Ü v V y Y z Z"

>> Concat( SortByColumns( Split( "d D e E f F g G h H i I j J k K l L m M n N o O p P r R s S t T u U v V y Y z Z Ç ç Ş ş Ü ü Ö ö İ ı Ğ ğ a A b B c C", " " ), "Value" ), Value, " " )
"a A b B c C ç Ç d D e E f F g G ğ Ğ h H i I İ ı j J k K l L m M n N o O ö Ö p P r R s S ş Ş t T u U ü Ü v V y Y z Z"

// REGULAR EXPRESSIONS
// Always uses invariant even though tr-TR is set, subject of https://github.com/microsoft/Power-Fx/issues/2538

// Results when using C# // Invariant tr-TR en-US

>> IsMatch( "İ", "i", MatchOptions.IgnoreCase ) // false TRUE TRUE
false

>> IsMatch( "i", "İ", MatchOptions.IgnoreCase ) // false TRUE TRUE
false

>> IsMatch( "ı", "I", MatchOptions.IgnoreCase ) // false TRUE false
false

>> IsMatch( "I", "ı", MatchOptions.IgnoreCase ) // false TRUE false
false

>> IsMatch( "İ", "I", MatchOptions.IgnoreCase ) // false false TRUE
false

>> IsMatch( "I", "İ", MatchOptions.IgnoreCase ) // false false TRUE
false

>> IsMatch( "ı", "i", MatchOptions.IgnoreCase ) // false false false
false

>> IsMatch( "i", "ı", MatchOptions.IgnoreCase ) // false false false
false

>> IsMatch( "i", "I", MatchOptions.IgnoreCase ) // TRUE false TRUE
true

>> IsMatch( "I", "i", MatchOptions.IgnoreCase ) // TRUE false TRUE
true

>> IsMatch( "ı", "İ", MatchOptions.IgnoreCase ) // false false false
false

>> IsMatch( "İ", "ı", MatchOptions.IgnoreCase ) // false false false
false

>> Match( "hiIıİİıIhi", "\u0130+" )
{FullMatch:"İİ",StartMatch:5,SubMatches:Table()}

>> IsMatch( "Sıgh", "\u0131", MatchOptions.Contains )
true
Loading
Loading