Skip to content

Commit

Permalink
Replace use of target dependent TestZ intrinsic (#104488)
Browse files Browse the repository at this point in the history 
* Replace uses of `TestZ` intrinsic

* remove pragma warning disable IntrinsicsInSystemPrivateCoreLibConditionParsing

* refactor control flow

---------

Co-authored-by: Tanner Gooding <[email protected]>
  • Loading branch information
@xtqqczze @tannergooding
xtqqczze and tannergooding authored Nov 27, 2024
1 parent 9f1494a commit cf7a744
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 46 deletions.
2 changes: 1 addition & 1 deletion ...raries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64DecoderHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1331,7 +1331,7 @@ public bool TryDecode256Core(
Vector256<sbyte> hi = Avx2.Shuffle(lutHigh, hiNibbles);
Vector256<sbyte> lo = Avx2.Shuffle(lutLow, loNibbles);

if (!Avx.TestZ(lo, hi))
if ((lo & hi) != Vector256<sbyte>.Zero)
{
result = default;
return false;
Expand Down
59 changes: 23 additions & 36 deletions src/libraries/System.Private.CoreLib/src/System/Text/Ascii.Utility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1501,10 +1501,10 @@ private static bool VectorContainsNonAsciiChar(Vector128<byte> asciiVector)
{
// max ASCII character is 0b_0111_1111, so the most significant bit (0x80) tells whether it contains non ascii

// prefer architecture specific intrinsic as they offer better perf
// For performance, prefer architecture specific implementation
if (Sse41.IsSupported)
{
return !Sse41.TestZ(asciiVector, Vector128.Create((byte)0x80));
return (asciiVector & Vector128.Create((byte)0x80)) != Vector128<byte>.Zero;
}
else if (AdvSimd.Arm64.IsSupported)
{
Expand All @@ -1520,23 +1520,21 @@ private static bool VectorContainsNonAsciiChar(Vector128<byte> asciiVector)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
{
// prefer architecture specific intrinsic as they offer better perf
if (Sse2.IsSupported)
// For performance, prefer architecture specific implementation
if (Sse41.IsSupported)
{
if (Sse41.IsSupported)
{
Vector128<ushort> asciiMaskForTestZ = Vector128.Create((ushort)0xFF80);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return !Sse41.TestZ(utf16Vector.AsInt16(), asciiMaskForTestZ.AsInt16());
}
else
{
Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
// The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
// has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
// to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
return (Sse2.MoveMask(Sse2.AddSaturate(utf16Vector, asciiMaskForAddSaturate).AsByte()) & 0b_1010_1010_1010_1010) != 0;
}
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector128<ushort> zeroIsAscii = utf16Vector & Vector128.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector128<ushort>.Zero;
}
else if (Sse2.IsSupported)
{
Vector128<ushort> asciiMaskForAddSaturate = Vector128.Create((ushort)0x7F80);
// The operation below forces the 0x8000 bit of each WORD to be set iff the WORD element
// has value >= 0x0800 (non-ASCII). Then we'll treat the vector as a BYTE vector in order
// to extract the mask. Reminder: the 0x0080 bit of each WORD should be ignored.
return (Sse2.MoveMask(Sse2.AddSaturate(utf16Vector, asciiMaskForAddSaturate).AsByte()) & 0b_1010_1010_1010_1010) != 0;
}
else if (AdvSimd.Arm64.IsSupported)
{
Expand All @@ -1557,18 +1555,10 @@ internal static bool VectorContainsNonAsciiChar(Vector128<ushort> utf16Vector)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static bool VectorContainsNonAsciiChar(Vector256<ushort> utf16Vector)
{
if (Avx.IsSupported)
{
Vector256<ushort> asciiMaskForTestZ = Vector256.Create((ushort)0xFF80);
return !Avx.TestZ(utf16Vector.AsInt16(), asciiMaskForTestZ.AsInt16());
}
else
{
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector256<ushort> zeroIsAscii = utf16Vector & Vector256.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector256<ushort>.Zero;
}
const ushort asciiMask = ushort.MaxValue - 127; // 0xFF80
Vector256<ushort> zeroIsAscii = utf16Vector & Vector256.Create(asciiMask);
// If a non-ASCII bit is set in any WORD of the vector, we have seen non-ASCII data.
return zeroIsAscii != Vector256<ushort>.Zero;
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
Expand Down Expand Up @@ -1601,14 +1591,13 @@ private static bool AllCharsInVectorAreAscii<T>(Vector128<T> vector)
if (typeof(T) == typeof(byte))
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsByte(), Vector128.Create((byte)0x80)) :
Sse41.IsSupported ? (vector.AsByte() & Vector128.Create((byte)0x80)) == Vector128<byte>.Zero :
AdvSimd.Arm64.IsSupported ? AllBytesInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsByte(), vector.AsByte()).AsUInt64().ToScalar()) :
vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return
Sse41.IsSupported ? Sse41.TestZ(vector.AsUInt16(), Vector128.Create((ushort)0xFF80)) :
AdvSimd.Arm64.IsSupported ? AllCharsInUInt64AreAscii(AdvSimd.Arm64.MaxPairwise(vector.AsUInt16(), vector.AsUInt16()).AsUInt64().ToScalar()) :
(vector.AsUInt16() & Vector128.Create((ushort)0xFF80)) == Vector128<ushort>.Zero;
}
Expand All @@ -1624,14 +1613,12 @@ private static bool AllCharsInVectorAreAscii<T>(Vector256<T> vector)
if (typeof(T) == typeof(byte))
{
return
Avx.IsSupported ? Avx.TestZ(vector.AsByte(), Vector256.Create((byte)0x80)) :
Avx.IsSupported ? (vector.AsByte() & Vector256.Create((byte)0x80)) == Vector256<byte>.Zero:
vector.AsByte().ExtractMostSignificantBits() == 0;
}
else
{
return
Avx.IsSupported ? Avx.TestZ(vector.AsUInt16(), Vector256.Create((ushort)0xFF80)) :
(vector.AsUInt16() & Vector256.Create((ushort)0xFF80)) == Vector256<ushort>.Zero;
return (vector.AsUInt16() & Vector256.Create((ushort)0xFF80)) == Vector256<ushort>.Zero;
}
}

Expand Down
16 changes: 8 additions & 8 deletions src/libraries/System.Private.CoreLib/src/System/Text/Latin1Utility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(combinedVector, latin1MaskForTestZ))
if ((combinedVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstOrSecondVector;
}
Expand Down Expand Up @@ -312,7 +312,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -347,7 +347,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
{
// If a non-Latin-1 bit is set in any WORD of the combined vector, we have seen non-Latin-1 data.
// Jump to the non-Latin-1 handler to figure out which particular vector contained non-Latin-1 data.
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -381,7 +381,7 @@ private static unsafe nuint GetIndexOfFirstNonLatin1Char_Sse2(char* pBuffer, nui
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(firstVector, latin1MaskForTestZ))
if ((firstVector & latin1MaskForTestZ) != Vector128<ushort>.Zero)
{
goto FoundNonLatin1DataInFirstVector;
}
Expand Down Expand Up @@ -795,7 +795,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
return 0;
}
Expand Down Expand Up @@ -837,7 +837,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto Finish;
}
Expand Down Expand Up @@ -878,7 +878,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(combinedVector, latin1MaskForTestZ))
if ((combinedVector & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto FoundNonLatin1DataInLoop;
}
Expand Down Expand Up @@ -914,7 +914,7 @@ private static unsafe nuint NarrowUtf16ToLatin1_Sse2(char* pUtf16Buffer, byte* p
if (Sse41.IsSupported)
#pragma warning restore IntrinsicsInSystemPrivateCoreLibAttributeNotSpecificEnough
{
if (!Sse41.TestZ(utf16VectorFirst, latin1MaskForTestZ))
if ((utf16VectorFirst & latin1MaskForTestZ) != Vector128<short>.Zero)
{
goto Finish; // found non-Latin-1 data
}
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Private.CoreLib/src/System/Text/Unicode/Utf8Utility.Transcoding.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -968,7 +968,7 @@ public static OperationStatus TranscodeToUtf8(char* pInputBuffer, int inputLengt
}
else if (Sse41.IsSupported)
{
if (!Sse41.TestZ(utf16Data, nonAsciiUtf16DataMask))
if ((utf16Data & nonAsciiUtf16DataMask) != Vector128<short>.Zero)
{
goto LoopTerminatedDueToNonAsciiDataInVectorLocal;
}
Expand Down

0 comments on commit cf7a744

Please sign in to comment.