diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 4d31cebad96b3..ea22a35bc31ff 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -2286,10 +2286,16 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } #if defined(FEATURE_MASKED_HW_INTRINSICS) && defined(TARGET_ARM64) + auto convertToMaskIfNeeded = [&](GenTree*& op) { + if (!varTypeIsMask(op)) + { + op = gtNewSimdCvtVectorToMaskNode(TYP_MASK, op, simdBaseJitType, simdSize); + } + }; + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrinsic)) { assert(numArgs > 0); - GenTree* op1 = retNode->AsHWIntrinsic()->Op(1); switch (intrinsic) { @@ -2304,14 +2310,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_TestFirstTrue: case NI_Sve_TestLastTrue: { - GenTree* op2 = retNode->AsHWIntrinsic()->Op(2); - // HWInstrinsic requires a mask for op2 - if (!varTypeIsMask(op2)) - { - retNode->AsHWIntrinsic()->Op(2) = - gtNewSimdCvtVectorToMaskNode(TYP_MASK, op2, simdBaseJitType, simdSize); - } + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(2)); break; } @@ -2324,14 +2324,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case NI_Sve_CreateBreakAfterPropagateMask: case NI_Sve_CreateBreakBeforePropagateMask: { - GenTree* op3 = retNode->AsHWIntrinsic()->Op(3); - // HWInstrinsic requires a mask for op3 - if (!varTypeIsMask(op3)) - { - retNode->AsHWIntrinsic()->Op(3) = - gtNewSimdCvtVectorToMaskNode(TYP_MASK, op3, simdBaseJitType, simdSize); - } + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(3)); break; } @@ -2339,11 +2333,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, break; } - if (!varTypeIsMask(op1)) - { - // Op1 input is a vector. HWInstrinsic requires a mask. - retNode->AsHWIntrinsic()->Op(1) = gtNewSimdCvtVectorToMaskNode(TYP_MASK, op1, simdBaseJitType, simdSize); - } + // HWInstrinsic requires a mask for op1 + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(1)); if (HWIntrinsicInfo::IsMultiReg(intrinsic)) { @@ -2354,6 +2345,22 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } } + if (HWIntrinsicInfo::IsEmbeddedMaskedOperation(intrinsic)) + { + switch (intrinsic) + { + case NI_Sve_CreateBreakPropagateMask: + { + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(1)); + convertToMaskIfNeeded(retNode->AsHWIntrinsic()->Op(2)); + break; + } + + default: + break; + } + } + if (retType != nodeRetType) { // HWInstrinsic returns a mask, but all returns must be vectors, so convert mask to vector. diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index f8e77197916df..1fb5272ab4a30 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -670,6 +670,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; bool hasShift = false; + insOpts embOpt = opt; switch (intrinEmbMask.id) { case NI_Sve_ShiftLeftLogical: @@ -689,6 +690,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) hasShift = true; break; + case NI_Sve_CreateBreakPropagateMask: + embOpt = INS_OPTS_SCALABLE_B; + break; + default: break; } @@ -699,13 +704,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op2, op2->AsHWIntrinsic()); for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) { - GetEmitter()->emitInsSve_R_R_I(insEmbMask, emitSize, reg1, reg2, helper.ImmValue(), opt, - sopt); + GetEmitter()->emitInsSve_R_R_I(insEmbMask, emitSize, reg1, reg2, helper.ImmValue(), + embOpt, sopt); } } else { - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, reg1, reg2, reg3, opt, sopt); + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, reg1, reg2, reg3, embOpt, sopt); } }; @@ -714,12 +719,25 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the // destination using /Z. - assert(targetReg != embMaskOp2Reg); - GetEmitter()->emitIns_R_R_R(INS_sve_movprfx, emitSize, targetReg, maskReg, embMaskOp1Reg, opt); + switch (intrinEmbMask.id) + { + case NI_Sve_CreateBreakPropagateMask: + assert(targetReg != embMaskOp1Reg); + GetEmitter()->emitIns_Mov(INS_sve_mov, emitSize, targetReg, embMaskOp2Reg, + /* canSkip */ true); + emitInsHelper(targetReg, maskReg, embMaskOp1Reg); + break; - // Finally, perform the actual "predicated" operation so that `targetReg` is the first operand - // and `embMaskOp2Reg` is the second operand. - emitInsHelper(targetReg, maskReg, embMaskOp2Reg); + default: + assert(targetReg != embMaskOp2Reg); + GetEmitter()->emitIns_R_R_R(INS_sve_movprfx, emitSize, targetReg, maskReg, + embMaskOp1Reg, opt); + + // Finally, perform the actual "predicated" operation so that `targetReg` is the first + // operand and `embMaskOp2Reg` is the second operand. + emitInsHelper(targetReg, maskReg, embMaskOp2Reg); + break; + } } else if (targetReg != falseReg) { diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 961076326818a..029d624379545 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -62,6 +62,7 @@ HARDWARE_INTRINSIC(Sve, CreateBreakAfterMask, HARDWARE_INTRINSIC(Sve, CreateBreakAfterPropagateMask, -1, 3, true, {INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_sve_brkpa, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, CreateBreakBeforeMask, -1, 2, true, {INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_sve_brkb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, CreateBreakBeforePropagateMask, -1, 3, true, {INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_sve_brkpb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve, CreateBreakPropagateMask, -1, -1, false, {INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_sve_brkn, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, CreateFalseMaskByte, -1, 0, false, {INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskDouble, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 266cb47537d50..c76cfe2031f58 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1642,7 +1642,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } else { - SingleTypeRegSet predMask = RBM_ALLMASK.GetPredicateRegSet(); + bool tgtPrefEmbOp2 = false; + SingleTypeRegSet predMask = RBM_ALLMASK.GetPredicateRegSet(); if (intrin.id == NI_Sve_ConditionalSelect) { // If this is conditional select, make sure to check the embedded @@ -1658,6 +1659,15 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { predMask = RBM_LOWMASK.GetPredicateRegSet(); } + + // Special-case, CreateBreakPropagateMask's op2 is the RMW node. + if (intrinEmb.id == NI_Sve_CreateBreakPropagateMask) + { + assert(embOp2Node->isRMWHWIntrinsic(compiler)); + assert(!tgtPrefOp1); + assert(!tgtPrefOp2); + tgtPrefEmbOp2 = true; + } } } else if (HWIntrinsicInfo::IsLowMaskedOperation(intrin.id)) @@ -1665,9 +1675,10 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou predMask = RBM_LOWMASK.GetPredicateRegSet(); } - if (tgtPrefOp2) + if (tgtPrefOp2 || tgtPrefEmbOp2) { - srcCount += BuildDelayFreeUses(intrin.op1, intrin.op2, predMask); + assert(!tgtPrefOp1); + srcCount += BuildDelayFreeUses(intrin.op1, nullptr, predMask); } else { @@ -1983,15 +1994,26 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou break; } - tgtPrefUse = BuildUse(embOp2Node->Op(1)); - srcCount += 1; - - for (size_t argNum = 2; argNum <= numArgs; argNum++) + size_t prefUseOpNum = 1; + if (intrinEmb.id == NI_Sve_CreateBreakPropagateMask) + { + prefUseOpNum = 2; + } + GenTree* prefUseNode = embOp2Node->Op(prefUseOpNum); + for (size_t argNum = 1; argNum <= numArgs; argNum++) { - srcCount += BuildDelayFreeUses(embOp2Node->Op(argNum), embOp2Node->Op(1)); + if (argNum == prefUseOpNum) + { + tgtPrefUse = BuildUse(prefUseNode); + srcCount += 1; + } + else + { + srcCount += BuildDelayFreeUses(embOp2Node->Op(argNum), prefUseNode); + } } - srcCount += BuildDelayFreeUses(intrin.op3, embOp2Node->Op(1)); + srcCount += BuildDelayFreeUses(intrin.op3, prefUseNode); } } else if (intrin.op2 != nullptr) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 3d52204ba67f2..98ba6df6d7a2b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -2288,6 +2288,55 @@ internal Arm64() { } public static unsafe Vector CreateBreakBeforePropagateMask(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) { throw new PlatformNotSupportedException(); } + + /// Set all predicate elements to false /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index e7dc14f3a3f2c..a781b001d8f30 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -2346,6 +2346,55 @@ internal Arm64() { } public static unsafe Vector CreateBreakBeforePropagateMask(Vector mask, Vector left, Vector right) => CreateBreakBeforePropagateMask(mask, left, right); + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// + /// svbool_t svbrkn[_b]_z(svbool_t pg, svbool_t op1, svbool_t op2) + /// BRKN Ptied2.B, Pg/Z, Pop1.B, Ptied2.B + /// + public static unsafe Vector CreateBreakPropagateMask(Vector totalMask, Vector fromMask) => CreateBreakPropagateMask(totalMask, fromMask); + + /// Set all predicate elements to false /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index f11418a9cbcc8..1ef08ed054901 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4651,6 +4651,15 @@ internal Arm64() { } public static System.Numerics.Vector CreateBreakBeforePropagateMask(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector CreateBreakBeforePropagateMask(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static unsafe System.Numerics.Vector CreateBreakPropagateMask(System.Numerics.Vector totalMask, System.Numerics.Vector fromMask) { throw null; } + public static System.Numerics.Vector CreateFalseMaskByte() { throw null; } public static System.Numerics.Vector CreateFalseMaskDouble() { throw null; } public static System.Numerics.Vector CreateFalseMaskInt16() { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index c20d6c8dd1bf5..0bad7a0e76653 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3332,7 +3332,16 @@ ("SveVecTernOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakBeforePropagateMask_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakBeforePropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["Op3BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt16()", ["NextValueOp2"] = "Helpers.getMaskInt16()", ["NextValueOp3"] = "Helpers.getMaskInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakBeforePropagateMask(firstOp, secondOp, thirdOp))", ["GetVectorResult"] = "Helpers.CreateBreakBeforePropagateMask(first, second, third)"}), ("SveVecTernOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakBeforePropagateMask_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakBeforePropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "Helpers.getMaskInt32()", ["NextValueOp3"] = "Helpers.getMaskInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakBeforePropagateMask(firstOp, secondOp, thirdOp))", ["GetVectorResult"] = "Helpers.CreateBreakBeforePropagateMask(first, second, third)"}), ("SveVecTernOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakBeforePropagateMask_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakBeforePropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["Op3BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "Helpers.getMaskInt64()", ["NextValueOp3"] = "Helpers.getMaskInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakBeforePropagateMask(firstOp, secondOp, thirdOp))", ["GetVectorResult"] = "Helpers.CreateBreakBeforePropagateMask(first, second, third)"}), - + + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()", ["NextValueOp2"] = "Helpers.getMaskByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()", ["NextValueOp2"] = "Helpers.getMaskUInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "Helpers.getMaskUInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "Helpers.getMaskUInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSByte()", ["NextValueOp2"] = "Helpers.getMaskSByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt16()", ["NextValueOp2"] = "Helpers.getMaskInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "Helpers.getMaskInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateBreakPropagateMask_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateBreakPropagateMask", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "Helpers.getMaskInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateBreakPropagateMask(left, right))", ["GetVectorResult"] = "Helpers.CreateBreakPropagateMask(left, right)"}), + ("ScalarBinOpRetVecTest.template",new Dictionary {["TestName"] = "Sve_CreateWhileLessThanMask16Bit_Int32", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateWhileLessThanMask16Bit", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "Helpers.WhileLessThanMask(left + (Int32)i, right) != (Int32)result[i]",}), ("ScalarBinOpRetVecTest.template",new Dictionary {["TestName"] = "Sve_CreateWhileLessThanMask16Bit_Int64", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateWhileLessThanMask16Bit", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "Helpers.WhileLessThanMask(left + (Int64)i, right) != (Int64)result[i]",}), ("ScalarBinOpRetVecTest.template",new Dictionary {["TestName"] = "Sve_CreateWhileLessThanMask16Bit_UInt32", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateWhileLessThanMask16Bit", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "Helpers.WhileLessThanMask(left + (UInt32)i, right) != (UInt32)result[i]",}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 1e3b4c342b1a3..9d4f4dfc67cda 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -8484,6 +8484,26 @@ public static T[] CreateBreakBeforePropagateMask(T[] mask, T[] op1, T[] op2) return result; } + public static T[] CreateBreakPropagateMask(T[] op1, T[] op2) where T : IBinaryInteger + { + var count = op1.Length; + var result = new T[count]; + + // embedded true mask + var mask = new T[count]; + for (var i = 0; i < count; i++) + { + mask[i] = T.One; + } + + if (LastActive(mask, op1) != T.Zero) + { + Array.Copy(op2, result, count); + } + + return result; + } + private static byte ConditionalExtract(byte[] op1, byte op2, byte[] op3, bool after) { int last = LastActiveElement(op1);