Arm64/Sve: Implement SVE Math *Multiply* APIs

src/coreclr/jit/emitarm64.cpp

@@ -4250,9 +4250,11 @@ void emitter::emitIns_Mov(
 
         case INS_sve_mov:
         {
-            if (isPredicateRegister(dstReg) && isPredicateRegister(srcReg))
+            // TODO-SVE: Remove check for insOptsNone() when predicate registers
+            //  are present.
+            if (insOptsNone(opt) && isPredicateRegister(dstReg) && isPredicateRegister(srcReg))
             {
-                assert(insOptsNone(opt));
+                // assert(insOptsNone(opt));
 
                 opt  = INS_OPTS_SCALABLE_B;
                 attr = EA_SCALABLE;
@@ -4263,6 +4265,16 @@ void emitter::emitIns_Mov(
                 }
                 fmt = IF_SVE_CZ_4A_L;
             }
+            else if (isVectorRegister(dstReg) && isVectorRegister(srcReg))
+            {
+                assert(insOptsScalable(opt));
+
+                if (IsRedundantMov(ins, size, dstReg, srcReg, canSkip))
+                {
+                    return;
+                }
+                fmt = IF_SVE_AU_3A;
+            }
             else
             {
                 unreached();

src/coreclr/jit/emitarm64sve.cpp

@@ -10374,7 +10374,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id)
         case IF_SVE_FN_3B:   // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long
         case IF_SVE_FO_3A:   // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate
         case IF_SVE_AT_3B:   // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated)
-        case IF_SVE_AU_3A:   // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
         case IF_SVE_BD_3B:   // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated)
         case IF_SVE_EF_3A:   // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product
         case IF_SVE_EI_3A:   // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product
@@ -10396,6 +10395,17 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id)
             dst += emitOutput_Instr(dst, code);
             break;
 
+        case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
+            code = emitInsCodeSve(ins, fmt);
+            code |= insEncodeReg_V<4, 0>(id->idReg1()); // ddddd
+            code |= insEncodeReg_V<9, 5>(id->idReg2()); // nnnnn
+            if (id->idIns() != INS_sve_mov)
+            {
+                code |= insEncodeReg_V<20, 16>(id->idReg3()); // mmmmm
+            }
+            dst += emitOutput_Instr(dst, code);
+            break;
+
         case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations
             code = emitInsCodeSve(ins, fmt);
             code |= insEncodeReg_V<4, 0>(id->idReg1());   // ddddd
@@ -12882,7 +12892,6 @@ void emitter::emitInsSveSanityCheck(instrDesc* id)
         case IF_SVE_FN_3B:   // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long
         case IF_SVE_FO_3A:   // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate
         case IF_SVE_AT_3B:   // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated)
-        case IF_SVE_AU_3A:   // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
         case IF_SVE_BD_3B:   // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated)
         case IF_SVE_EF_3A:   // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product
         case IF_SVE_EI_3A:   // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product
@@ -12902,6 +12911,12 @@ void emitter::emitInsSveSanityCheck(instrDesc* id)
             assert(isVectorRegister(id->idReg2())); // nnnnn/mmmmm
             assert(isVectorRegister(id->idReg3())); // mmmmm/aaaaa
             break;
+        case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
+            assert(insOptsScalable(id->idInsOpt()));
+            assert(isVectorRegister(id->idReg1()));                                 // ddddd
+            assert(isVectorRegister(id->idReg2()));                                 // nnnnn/mmmmm
+            assert((id->idIns() == INS_sve_mov) || isVectorRegister(id->idReg3())); // mmmmm/aaaaa
+            break;
 
         case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product
         case IF_SVE_EW_3A:   // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer)
@@ -14526,7 +14541,6 @@ void emitter::emitDispInsSveHelp(instrDesc* id)
         case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate
         // <Zd>.D, <Zn>.D, <Zm>.D
         case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated)
-        case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
         // <Zd>.B, <Zn>.B, <Zm>.B
         case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment)
         case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated)
@@ -14541,6 +14555,20 @@ void emitter::emitDispInsSveHelp(instrDesc* id)
             emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa
             break;
 
+        // <Zd>.D, <Zn>.D, <Zm>.D
+        case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated)
+            emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd
+            if (id->idIns() == INS_sve_mov)
+            {
+                emitDispSveReg(id->idReg2(), id->idInsOpt(), false); // nnnnn/mmmmm
+            }
+            else
+            {
+                emitDispSveReg(id->idReg2(), id->idInsOpt(), true);  // nnnnn/mmmmm
+                emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa
+            }
+            break;
+
         // <Zda>.D, <Zn>.D, <Zm>.D
         case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer)
         // <Zdn>.D, <Zm>.D, <Za>.D

src/coreclr/jit/gentree.cpp

@@ -27955,7 +27955,7 @@ bool GenTreeLclVar::IsNeverNegative(Compiler* comp) const
     return comp->lvaGetDesc(GetLclNum())->IsNeverNegative();
 }
 
-#if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS)
+#if (defined(TARGET_XARCH) || defined(TARGET_ARM64)) && defined(FEATURE_HW_INTRINSICS)
 //------------------------------------------------------------------------
 // GetResultOpNumForRmwIntrinsic: check if the result is written into one of the operands.
 // In the case that none of the operand is overwritten, check if any of them is lastUse.
@@ -27966,7 +27966,11 @@ bool GenTreeLclVar::IsNeverNegative(Compiler* comp) const
 //
 unsigned GenTreeHWIntrinsic::GetResultOpNumForRmwIntrinsic(GenTree* use, GenTree* op1, GenTree* op2, GenTree* op3)
 {
+#if defined(TARGET_XARCH)
     assert(HWIntrinsicInfo::IsFmaIntrinsic(gtHWIntrinsicId) || HWIntrinsicInfo::IsPermuteVar2x(gtHWIntrinsicId));
+#elif defined(TARGET_ARM64)
+    assert(HWIntrinsicInfo::IsFmaIntrinsic(gtHWIntrinsicId));
+#endif
 
     if (use != nullptr && use->OperIs(GT_STORE_LCL_VAR))
     {

src/coreclr/jit/hwintrinsic.h

@@ -216,27 +216,27 @@ enum HWIntrinsicFlag : unsigned int
     // The intrinsic is an RMW intrinsic
     HW_Flag_RmwIntrinsic = 0x1000000,
 
-    // The intrinsic is a FusedMultiplyAdd intrinsic
-    HW_Flag_FmaIntrinsic = 0x2000000,
-
     // The intrinsic is a PermuteVar2x intrinsic
-    HW_Flag_PermuteVar2x = 0x4000000,
+    HW_Flag_PermuteVar2x = 0x2000000,
 
     // The intrinsic is an embedded broadcast compatible intrinsic
-    HW_Flag_EmbBroadcastCompatible = 0x8000000,
+    HW_Flag_EmbBroadcastCompatible = 0x4000000,
 
     // The intrinsic is an embedded rounding compatible intrinsic
-    HW_Flag_EmbRoundingCompatible = 0x10000000,
+    HW_Flag_EmbRoundingCompatible = 0x8000000,
 
     // The intrinsic is an embedded masking compatible intrinsic
-    HW_Flag_EmbMaskingCompatible = 0x20000000,
+    HW_Flag_EmbMaskingCompatible = 0x10000000,
 #elif defined(TARGET_ARM64)
 
     // The intrinsic has an enum operand. Using this implies HW_Flag_HasImmediateOperand.
     HW_Flag_HasEnumOperand = 0x1000000,
 
 #endif // TARGET_XARCH
 
+    // The intrinsic is a FusedMultiplyAdd intrinsic
+    HW_Flag_FmaIntrinsic = 0x20000000,
+
     HW_Flag_CanBenefitFromConstantProp = 0x80000000,
 };
 
@@ -935,17 +935,17 @@ struct HWIntrinsicInfo
         return (flags & HW_Flag_MaybeNoJmpTableIMM) != 0;
     }
 
-#if defined(TARGET_XARCH)
-    static bool IsRmwIntrinsic(NamedIntrinsic id)
+    static bool IsFmaIntrinsic(NamedIntrinsic id)
     {
         HWIntrinsicFlag flags = lookupFlags(id);
-        return (flags & HW_Flag_RmwIntrinsic) != 0;
+        return (flags & HW_Flag_FmaIntrinsic) != 0;
     }
 
-    static bool IsFmaIntrinsic(NamedIntrinsic id)
+#if defined(TARGET_XARCH)
+    static bool IsRmwIntrinsic(NamedIntrinsic id)
     {
         HWIntrinsicFlag flags = lookupFlags(id);
-        return (flags & HW_Flag_FmaIntrinsic) != 0;
+        return (flags & HW_Flag_RmwIntrinsic) != 0;
     }
 
     static bool IsPermuteVar2x(NamedIntrinsic id)

src/coreclr/jit/hwintrinsicarm64.cpp

@@ -277,6 +277,8 @@ void HWIntrinsicInfo::lookupImmBounds(
             case NI_AdvSimd_Arm64_StoreSelectedScalarVector128x4:
             case NI_AdvSimd_Arm64_DuplicateSelectedScalarToVector128:
             case NI_AdvSimd_Arm64_InsertSelectedScalar:
+            case NI_Sve_FusedMultiplyAddBySelectedScalar:
+            case NI_Sve_FusedMultiplySubtractBySelectedScalar:
                 immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) - 1;
                 break;