From b6aa640a1b53055ca6af3167c247a4ebb6b6b9ce Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Mon, 13 May 2024 21:35:39 +0100 Subject: [PATCH] Refactor handling of two immediates in hwintrinsic (#102071) * Refactor handling of two immediates in hwintrinsic * Add immOp2 assert * Update comments for getHWIntrinsicImmTypes * fix arg name * update function summaries --- src/coreclr/jit/codegen.h | 2 +- src/coreclr/jit/compiler.h | 28 ++ src/coreclr/jit/hwintrinsic.cpp | 289 +++++++++----------- src/coreclr/jit/hwintrinsic.h | 2 +- src/coreclr/jit/hwintrinsicarm64.cpp | 140 +++++++++- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 16 +- src/coreclr/jit/hwintrinsicxarch.cpp | 23 ++ src/coreclr/jit/lsraarm64.cpp | 6 +- 8 files changed, 331 insertions(+), 175 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index ed1e4be7cf43e4..3aeaf5c44e3198 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1009,7 +1009,7 @@ class CodeGen final : public CodeGenInterface class HWIntrinsicImmOpHelper final { public: - HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTree* immOp, GenTreeHWIntrinsic* intrin); + HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTree* immOp, GenTreeHWIntrinsic* intrin, int immNum = 1); void EmitBegin(); void EmitCaseEnd(); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index b17ee58465b89c..01225550850ffb 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4603,6 +4603,34 @@ class Compiler NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immLowerBound, int immUpperBound); GenTree* addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound, int immUpperBound); + void getHWIntrinsicImmOps(NamedIntrinsic intrinsic, + CORINFO_SIG_INFO* sig, + GenTree** immOp1Ptr, + GenTree** immOp2Ptr); + + bool CheckHWIntrinsicImmRange(NamedIntrinsic intrinsic, + CorInfoType simdBaseJitType, + GenTree* immOp, + bool mustExpand, + int immLowerBound, + int immUpperBound, + bool hasFullRangeImm, + bool *useFallback); + +#if defined(TARGET_ARM64) + + void getHWIntrinsicImmTypes(NamedIntrinsic intrinsic, + CORINFO_SIG_INFO* sig, + unsigned immNumber, + var_types simdBaseType, + CorInfoType simdBaseJitType, + CORINFO_CLASS_HANDLE op2ClsHnd, + CORINFO_CLASS_HANDLE op3ClsHnd, + unsigned* immSimdSize, + var_types* immSimdBaseType); + +#endif // TARGET_ARM64 + #endif // FEATURE_HW_INTRINSICS GenTree* impArrayAccessIntrinsic(CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 3187c3e5c2e044..0a43bddbd34b19 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1032,6 +1032,97 @@ struct HWIntrinsicSignatureReader final } }; +//------------------------------------------------------------------------ +// CheckHWIntrinsicImmRange: Check if an immediate is within the valid range +// +// Arguments: +// intrinsicId -- HW intrinsic id +// simdBaseJitType -- The base JIT type of SIMD type of the intrinsic +// immOp -- Immediate to check is within range +// mustExpand -- true if the intrinsic must expand to a GenTree*; otherwise, false +// immLowerBound -- the lower valid bound of the immediate +// immLowerBound -- the upper valid bound of the immediate +// hasFullRangeImm -- the range has all valid values. The immediate is always within range. +// useFallback [OUT] -- Only set if false is returned. A fallback can be used instead. +// +// Return Value: +// returns true if immOp is within range. Otherwise false. +// +bool Compiler::CheckHWIntrinsicImmRange(NamedIntrinsic intrinsic, + CorInfoType simdBaseJitType, + GenTree* immOp, + bool mustExpand, + int immLowerBound, + int immUpperBound, + bool hasFullRangeImm, + bool* useFallback) +{ + *useFallback = false; + + if (!hasFullRangeImm && immOp->IsCnsIntOrI()) + { + const int ival = (int)immOp->AsIntCon()->IconValue(); + bool immOutOfRange; +#ifdef TARGET_XARCH + if (HWIntrinsicInfo::isAVX2GatherIntrinsic(intrinsic)) + { + immOutOfRange = (ival != 1) && (ival != 2) && (ival != 4) && (ival != 8); + } + else +#endif + { + immOutOfRange = (ival < immLowerBound) || (ival > immUpperBound); + } + + if (immOutOfRange) + { + assert(!mustExpand); + // The imm-HWintrinsics that do not accept all imm8 values may throw + // ArgumentOutOfRangeException when the imm argument is not in the valid range + return false; + } + } + else if (!immOp->IsCnsIntOrI()) + { + if (HWIntrinsicInfo::NoJmpTableImm(intrinsic)) + { + *useFallback = true; + return false; + } +#if defined(TARGET_XARCH) + else if (HWIntrinsicInfo::MaybeNoJmpTableImm(intrinsic)) + { +#if defined(TARGET_X86) + var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); + + if (varTypeIsLong(simdBaseType)) + { + if (!mustExpand) + { + return false; + } + } + else +#endif // TARGET_XARCH + { + *useFallback = true; + return false; + } + } +#endif // TARGET_XARCH + else if (!mustExpand) + { + // When the imm-argument is not a constant and we are not being forced to expand, we need to + // return false so a GT_CALL to the intrinsic method is emitted instead. The + // intrinsic method is recursive and will be forced to expand, at which point + // we emit some less efficient fallback code. + return false; + } + } + + return true; +} + //------------------------------------------------------------------------ // impHWIntrinsic: Import a hardware intrinsic as a GT_HWINTRINSIC node if possible // @@ -1162,190 +1253,64 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } var_types simdBaseType = TYP_UNKNOWN; - GenTree* immOp = nullptr; if (simdBaseJitType != CORINFO_TYPE_UNDEF) { simdBaseType = JitType2PreciseVarType(simdBaseJitType); } + const unsigned simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig); + HWIntrinsicSignatureReader sigReader; sigReader.Read(info.compCompHnd, sig); -#ifdef TARGET_ARM64 - if ((intrinsic == NI_AdvSimd_Insert) || (intrinsic == NI_AdvSimd_InsertScalar) || - ((intrinsic >= NI_AdvSimd_LoadAndInsertScalar) && (intrinsic <= NI_AdvSimd_LoadAndInsertScalarVector64x4)) || - ((intrinsic >= NI_AdvSimd_Arm64_LoadAndInsertScalar) && - (intrinsic <= NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4))) - { - assert(sig->numArgs == 3); - immOp = impStackTop(1).val; - assert(HWIntrinsicInfo::isImmOp(intrinsic, immOp)); - } - else if (intrinsic == NI_AdvSimd_Arm64_InsertSelectedScalar) - { - // InsertSelectedScalar intrinsic has two immediate operands. - // Since all the remaining intrinsics on both platforms have only one immediate - // operand, in order to not complicate the shared logic even further we ensure here that - // 1) The second immediate operand immOp2 is constant and - // 2) its value belongs to [0, sizeof(op3) / sizeof(op3.BaseType)). - // If either is false, we should fallback to the managed implementation Insert(dst, dstIdx, Extract(src, - // srcIdx)). - // The check for the first immediate operand immOp will use the same logic as other intrinsics that have an - // immediate operand. - - GenTree* immOp2 = nullptr; - - assert(sig->numArgs == 4); - - immOp = impStackTop(2).val; - immOp2 = impStackTop().val; - - assert(HWIntrinsicInfo::isImmOp(intrinsic, immOp)); - assert(HWIntrinsicInfo::isImmOp(intrinsic, immOp2)); - - if (!immOp2->IsCnsIntOrI()) - { - assert(HWIntrinsicInfo::NoJmpTableImm(intrinsic)); - return impNonConstFallback(intrinsic, retType, simdBaseJitType); - } - - unsigned int otherSimdSize = 0; - CorInfoType otherBaseJitType = getBaseJitTypeAndSizeOfSIMDType(sigReader.op3ClsHnd, &otherSimdSize); - var_types otherBaseType = JitType2PreciseVarType(otherBaseJitType); - - assert(otherBaseJitType == simdBaseJitType); - - int immLowerBound2 = 0; - int immUpperBound2 = 0; - - HWIntrinsicInfo::lookupImmBounds(intrinsic, otherSimdSize, otherBaseType, &immLowerBound2, &immUpperBound2); + GenTree* immOp1 = nullptr; + GenTree* immOp2 = nullptr; + int immLowerBound = 0; + int immUpperBound = 0; + bool hasFullRangeImm = false; + bool useFallback = false; - const int immVal2 = (int)immOp2->AsIntCon()->IconValue(); + getHWIntrinsicImmOps(intrinsic, sig, &immOp1, &immOp2); - if ((immVal2 < immLowerBound2) || (immVal2 > immUpperBound2)) + // Validate the second immediate +#ifdef TARGET_ARM64 + if (immOp2 != nullptr) + { + unsigned immSimdSize = simdSize; + var_types immSimdBaseType = simdBaseType; + getHWIntrinsicImmTypes(intrinsic, sig, 2, simdBaseType, simdBaseJitType, sigReader.op2ClsHnd, + sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); + HWIntrinsicInfo::lookupImmBounds(intrinsic, immSimdSize, immSimdBaseType, 2, &immLowerBound, &immUpperBound); + + if (!CheckHWIntrinsicImmRange(intrinsic, simdBaseJitType, immOp2, mustExpand, immLowerBound, immUpperBound, + false, &useFallback)) { - assert(!mustExpand); - return nullptr; + return useFallback ? impNonConstFallback(intrinsic, retType, simdBaseJitType) : nullptr; } } - else +#else + assert(immOp2 == nullptr); #endif - if ((sig->numArgs > 0) && HWIntrinsicInfo::isImmOp(intrinsic, impStackTop().val)) - { - // NOTE: The following code assumes that for all intrinsics - // taking an immediate operand, that operand will be last. - immOp = impStackTop().val; - } - - const unsigned simdSize = HWIntrinsicInfo::lookupSimdSize(this, intrinsic, sig); - - int immLowerBound = 0; - int immUpperBound = 0; - bool hasFullRangeImm = false; - if (immOp != nullptr) + // Validate the first immediate + if (immOp1 != nullptr) { -#ifdef TARGET_XARCH +#ifdef TARGET_ARM64 + unsigned immSimdSize = simdSize; + var_types immSimdBaseType = simdBaseType; + getHWIntrinsicImmTypes(intrinsic, sig, 1, simdBaseType, simdBaseJitType, sigReader.op2ClsHnd, + sigReader.op3ClsHnd, &immSimdSize, &immSimdBaseType); + HWIntrinsicInfo::lookupImmBounds(intrinsic, immSimdSize, immSimdBaseType, 1, &immLowerBound, &immUpperBound); +#else immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic); hasFullRangeImm = HWIntrinsicInfo::HasFullRangeImm(intrinsic); -#elif defined(TARGET_ARM64) - if (category == HW_Category_SIMDByIndexedElement) - { - CorInfoType indexedElementBaseJitType; - var_types indexedElementBaseType; - unsigned int indexedElementSimdSize = 0; - - if (numArgs == 3) - { - indexedElementBaseJitType = - getBaseJitTypeAndSizeOfSIMDType(sigReader.op2ClsHnd, &indexedElementSimdSize); - indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); - } - else - { - assert(numArgs == 4); - indexedElementBaseJitType = - getBaseJitTypeAndSizeOfSIMDType(sigReader.op3ClsHnd, &indexedElementSimdSize); - indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); - - if (intrinsic == NI_Dp_DotProductBySelectedQuadruplet) - { - assert(((simdBaseType == TYP_INT) && (indexedElementBaseType == TYP_BYTE)) || - ((simdBaseType == TYP_UINT) && (indexedElementBaseType == TYP_UBYTE))); - // The second source operand of sdot, udot instructions is an indexed 32-bit element. - indexedElementBaseJitType = simdBaseJitType; - indexedElementBaseType = simdBaseType; - } - } - - assert(indexedElementBaseType == simdBaseType); - HWIntrinsicInfo::lookupImmBounds(intrinsic, indexedElementSimdSize, simdBaseType, &immLowerBound, - &immUpperBound); - } - else - { - HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, simdBaseType, &immLowerBound, &immUpperBound); - } #endif - if (!hasFullRangeImm && immOp->IsCnsIntOrI()) + if (!CheckHWIntrinsicImmRange(intrinsic, simdBaseJitType, immOp1, mustExpand, immLowerBound, immUpperBound, + hasFullRangeImm, &useFallback)) { - const int ival = (int)immOp->AsIntCon()->IconValue(); - bool immOutOfRange; -#ifdef TARGET_XARCH - if (HWIntrinsicInfo::isAVX2GatherIntrinsic(intrinsic)) - { - immOutOfRange = (ival != 1) && (ival != 2) && (ival != 4) && (ival != 8); - } - else -#endif - { - immOutOfRange = (ival < immLowerBound) || (ival > immUpperBound); - } - - if (immOutOfRange) - { - assert(!mustExpand); - // The imm-HWintrinsics that do not accept all imm8 values may throw - // ArgumentOutOfRangeException when the imm argument is not in the valid range - return nullptr; - } - } - else if (!immOp->IsCnsIntOrI()) - { - if (HWIntrinsicInfo::NoJmpTableImm(intrinsic)) - { - return impNonConstFallback(intrinsic, retType, simdBaseJitType); - } -#if defined(TARGET_XARCH) - else if (HWIntrinsicInfo::MaybeNoJmpTableImm(intrinsic)) - { -#if defined(TARGET_X86) - var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); - - if (varTypeIsLong(simdBaseType)) - { - if (!mustExpand) - { - return nullptr; - } - } - else -#endif // TARGET_XARCH - { - return impNonConstFallback(intrinsic, retType, simdBaseJitType); - } - } -#endif // TARGET_XARCH - else if (!mustExpand) - { - // When the imm-argument is not a constant and we are not being forced to expand, we need to - // return nullptr so a GT_CALL to the intrinsic method is emitted instead. The - // intrinsic method is recursive and will be forced to expand, at which point - // we emit some less efficient fallback code. - return nullptr; - } + return useFallback ? impNonConstFallback(intrinsic, retType, simdBaseJitType) : nullptr; } } diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 1dec44d4e6df90..2f7690c091e9fa 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -514,7 +514,7 @@ struct HWIntrinsicInfo static int lookupImmUpperBound(NamedIntrinsic intrinsic); #elif defined(TARGET_ARM64) static void lookupImmBounds( - NamedIntrinsic intrinsic, int simdSize, var_types baseType, int* lowerBound, int* upperBound); + NamedIntrinsic intrinsic, int simdSize, var_types baseType, int immNumber, int* lowerBound, int* upperBound); #else #error Unsupported platform #endif diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 84b7e6b31387d0..e11f8751777081 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -210,6 +210,141 @@ bool HWIntrinsicInfo::isScalarIsa(CORINFO_InstructionSet isa) } } +//------------------------------------------------------------------------ +// getHWIntrinsicImmOps: Gets the immediate Ops for an intrinsic +// +// Arguments: +// intrinsic -- NamedIntrinsic associated with the HWIntrinsic to lookup +// sig -- signature of the intrinsic call. +// immOp1Ptr [OUT] -- The first immediate Op +// immOp2Ptr [OUT] -- The second immediate Op, if any. Otherwise unchanged. +// +void Compiler::getHWIntrinsicImmOps(NamedIntrinsic intrinsic, + CORINFO_SIG_INFO* sig, + GenTree** immOp1Ptr, + GenTree** immOp2Ptr) +{ + if (!HWIntrinsicInfo::HasImmediateOperand(intrinsic)) + { + return; + } + + // Position of the immediates from top of stack + int imm1Pos = -1; + int imm2Pos = -1; + + switch (intrinsic) + { + case NI_AdvSimd_Insert: + case NI_AdvSimd_InsertScalar: + case NI_AdvSimd_LoadAndInsertScalar: + case NI_AdvSimd_LoadAndInsertScalarVector64x2: + case NI_AdvSimd_LoadAndInsertScalarVector64x3: + case NI_AdvSimd_LoadAndInsertScalarVector64x4: + case NI_AdvSimd_Arm64_LoadAndInsertScalar: + case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2: + case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3: + case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4: + assert(sig->numArgs == 3); + imm1Pos = 1; + break; + + case NI_AdvSimd_Arm64_InsertSelectedScalar: + assert(sig->numArgs == 4); + imm1Pos = 2; + imm2Pos = 0; + break; + + default: + assert(sig->numArgs > 0); + imm1Pos = 0; + break; + } + + if (imm1Pos >= 0) + { + *immOp1Ptr = impStackTop(imm1Pos).val; + assert(HWIntrinsicInfo::isImmOp(intrinsic, *immOp1Ptr)); + } + + if (imm2Pos >= 0) + { + *immOp2Ptr = impStackTop(imm2Pos).val; + assert(HWIntrinsicInfo::isImmOp(intrinsic, *immOp2Ptr)); + } +} + +//------------------------------------------------------------------------ +// getHWIntrinsicImmTypes: Gets the type/size for an immediate for an intrinsic +// if it differs from the default type/size of the instrinsic +// +// Arguments: +// intrinsic -- NamedIntrinsic associated with the HWIntrinsic to lookup +// sig -- signature of the intrinsic call. +// immNumber -- Which immediate to use (1 for most intrinsics) +// simdBaseType -- base type of the intrinsic +// simdType -- vector size of the intrinsic +// op2ClsHnd -- cls handler for op2 +// op2ClsHnd -- cls handler for op3 +// immSimdSize [IN/OUT] -- Size of the immediate to override +// immSimdBaseType [IN/OUT] -- Base type of the immediate to override +// +void Compiler::getHWIntrinsicImmTypes(NamedIntrinsic intrinsic, + CORINFO_SIG_INFO* sig, + unsigned immNumber, + var_types simdBaseType, + CorInfoType simdBaseJitType, + CORINFO_CLASS_HANDLE op2ClsHnd, + CORINFO_CLASS_HANDLE op3ClsHnd, + unsigned* immSimdSize, + var_types* immSimdBaseType) +{ + HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); + + if (category == HW_Category_SIMDByIndexedElement) + { + assert(immNumber == 1); + + CorInfoType indexedElementBaseJitType; + var_types indexedElementBaseType; + *immSimdSize = 0; + + if (sig->numArgs == 3) + { + indexedElementBaseJitType = getBaseJitTypeAndSizeOfSIMDType(op2ClsHnd, immSimdSize); + indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); + } + else + { + assert(sig->numArgs == 4); + indexedElementBaseJitType = getBaseJitTypeAndSizeOfSIMDType(op3ClsHnd, immSimdSize); + indexedElementBaseType = JitType2PreciseVarType(indexedElementBaseJitType); + + if (intrinsic == NI_Dp_DotProductBySelectedQuadruplet) + { + assert(((simdBaseType == TYP_INT) && (indexedElementBaseType == TYP_BYTE)) || + ((simdBaseType == TYP_UINT) && (indexedElementBaseType == TYP_UBYTE))); + // The second source operand of sdot, udot instructions is an indexed 32-bit element. + indexedElementBaseType = simdBaseType; + } + } + + assert(indexedElementBaseType == simdBaseType); + } + else if (intrinsic == NI_AdvSimd_Arm64_InsertSelectedScalar) + { + if (immNumber == 2) + { + CorInfoType otherBaseJitType = getBaseJitTypeAndSizeOfSIMDType(op3ClsHnd, immSimdSize); + *immSimdBaseType = JitType2PreciseVarType(otherBaseJitType); + assert(otherBaseJitType == simdBaseJitType); + } + // For imm1 use default simd sizes. + } + + // For all other imms, use default simd sizes +} + //------------------------------------------------------------------------ // lookupImmBounds: Gets the lower and upper bounds for the imm-value of a given NamedIntrinsic // @@ -217,11 +352,12 @@ bool HWIntrinsicInfo::isScalarIsa(CORINFO_InstructionSet isa) // intrinsic -- NamedIntrinsic associated with the HWIntrinsic to lookup // simdType -- vector size // baseType -- base type of the Vector64/128 +// immNumber -- which immediate operand to check for (most intrinsics only have one) // pImmLowerBound [OUT] - The lower incl. bound for a value of the intrinsic immediate operand // pImmUpperBound [OUT] - The upper incl. bound for a value of the intrinsic immediate operand // void HWIntrinsicInfo::lookupImmBounds( - NamedIntrinsic intrinsic, int simdSize, var_types baseType, int* pImmLowerBound, int* pImmUpperBound) + NamedIntrinsic intrinsic, int simdSize, var_types baseType, int immNumber, int* pImmLowerBound, int* pImmUpperBound) { HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); bool hasImmediateOperand = HasImmediateOperand(intrinsic); @@ -1909,7 +2045,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, } assert(HWIntrinsicInfo::isImmOp(intrinsic, op3)); - HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, simdBaseType, &immLowerBound, &immUpperBound); + HWIntrinsicInfo::lookupImmBounds(intrinsic, simdSize, simdBaseType, 1, &immLowerBound, &immUpperBound); op3 = addRangeCheckIfNeeded(intrinsic, op3, (!op3->IsCnsIntOrI()), immLowerBound, immUpperBound); argType = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg1, &argClass))); op1 = getArgForHWIntrinsic(argType, argClass); diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6d381680cf4ab3..c0dcd95b5b1b9e 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -20,9 +20,10 @@ // intrinsic node. The register will be later used to store computed branch target address. // // Arguments: -// codeGen -- an instance of CodeGen class. -// immOp -- an immediate operand of the intrinsic. -// intrin -- a hardware intrinsic tree node. +// codeGen -- an instance of CodeGen class. +// immOp -- an immediate operand of the intrinsic. +// intrin -- a hardware intrinsic tree node. +// immNumber -- which immediate operand to use (most intrinsics only have one). // // Note: This class is designed to be used in the following way // HWIntrinsicImmOpHelper helper(this, immOp, intrin); @@ -35,7 +36,10 @@ // This allows to combine logic for cases when immOp->isContainedIntOrIImmed() is either true or false in a form // of a for-loop. // -CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTree* immOp, GenTreeHWIntrinsic* intrin) +CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, + GenTree* immOp, + GenTreeHWIntrinsic* intrin, + int immNumber /* = 1 */) : codeGen(codeGen) , endLabel(nullptr) , nonZeroLabel(nullptr) @@ -75,12 +79,12 @@ CodeGen::HWIntrinsicImmOpHelper::HWIntrinsicImmOpHelper(CodeGen* codeGen, GenTre const unsigned int indexedElementSimdSize = genTypeSize(indexedElementOpType); HWIntrinsicInfo::lookupImmBounds(intrin->GetHWIntrinsicId(), indexedElementSimdSize, - intrin->GetSimdBaseType(), &immLowerBound, &immUpperBound); + intrin->GetSimdBaseType(), immNumber, &immLowerBound, &immUpperBound); } else { HWIntrinsicInfo::lookupImmBounds(intrin->GetHWIntrinsicId(), intrin->GetSimdSize(), - intrin->GetSimdBaseType(), &immLowerBound, &immUpperBound); + intrin->GetSimdBaseType(), immNumber, &immLowerBound, &immUpperBound); } nonConstImmReg = immOp->GetRegNum(); diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 19062dde2d7b77..524b54c00616d3 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -4436,4 +4436,27 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, return retNode; } + +//------------------------------------------------------------------------ +// getHWIntrinsicImmOps: Gets the immediate Ops for an intrinsic +// +// Arguments: +// intrinsic -- NamedIntrinsic associated with the HWIntrinsic to lookup +// sig -- signature of the intrinsic call. +// immOp1Ptr [OUT] -- The first immediate Op +// immOp2Ptr [OUT] -- The second immediate Op, if any. Otherwise unchanged. +// +void Compiler::getHWIntrinsicImmOps(NamedIntrinsic intrinsic, + CORINFO_SIG_INFO* sig, + GenTree** immOp1Ptr, + GenTree** immOp2Ptr) +{ + if ((sig->numArgs > 0) && HWIntrinsicInfo::isImmOp(intrinsic, impStackTop().val)) + { + // NOTE: The following code assumes that for all intrinsics + // taking an immediate operand, that operand will be last. + *immOp1Ptr = impStackTop().val; + } +} + #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index e548fc3feec669..264bc70b0a74de 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1383,13 +1383,13 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou assert(varTypeIsSIMD(indexedElementOpType)); const unsigned int indexedElementSimdSize = genTypeSize(indexedElementOpType); - HWIntrinsicInfo::lookupImmBounds(intrin.id, indexedElementSimdSize, intrin.baseType, &immLowerBound, + HWIntrinsicInfo::lookupImmBounds(intrin.id, indexedElementSimdSize, intrin.baseType, 1, &immLowerBound, &immUpperBound); } else { - HWIntrinsicInfo::lookupImmBounds(intrin.id, intrinsicTree->GetSimdSize(), intrin.baseType, &immLowerBound, - &immUpperBound); + HWIntrinsicInfo::lookupImmBounds(intrin.id, intrinsicTree->GetSimdSize(), intrin.baseType, 1, + &immLowerBound, &immUpperBound); } if ((immLowerBound != 0) || (immUpperBound != 1))