From 40d86439da808cd13fec4dbe44d8e9352cd9352d Mon Sep 17 00:00:00 2001 From: "Jin,Zhonghui" Date: Tue, 7 Nov 2023 01:18:09 +0800 Subject: [PATCH] [Backport] 8256973: Intrinsic creation for VectorMask query (lastTrue,firstTrue,trueCount) APIs Summary: [Backport] 8256973: Intrinsic creation for VectorMask query (lastTrue,firstTrue,trueCount) APIs Test Plan: ci jtreg Reviewed-by: JoshuaZhuwj Issue: https://github.com/dragonwell-project/dragonwell11/issues/615 --- mask.incr | 43 +++++ src/hotspot/cpu/x86/assembler_x86.cpp | 8 + src/hotspot/cpu/x86/assembler_x86.hpp | 2 + src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp | 51 ++++++ src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp | 10 ++ src/hotspot/cpu/x86/x86.ad | 69 ++++++++ src/hotspot/share/classfile/vmSymbols.hpp | 7 +- src/hotspot/share/opto/c2compiler.cpp | 1 + src/hotspot/share/opto/classes.hpp | 4 + src/hotspot/share/opto/library_call.cpp | 2 + src/hotspot/share/opto/library_call.hpp | 1 + src/hotspot/share/opto/vectorIntrinsics.cpp | 54 ++++++ src/hotspot/share/opto/vectornode.cpp | 15 ++ src/hotspot/share/opto/vectornode.hpp | 39 +++++ src/hotspot/share/prims/vectorSupport.cpp | 36 ++++ src/hotspot/share/prims/vectorSupport.hpp | 7 +- .../jdk/internal/vm/vector/VectorSupport.java | 19 +++ .../jdk/incubator/vector/AbstractMask.java | 55 +++--- .../jdk/incubator/vector/Byte128Vector.java | 23 +++ .../jdk/incubator/vector/Byte256Vector.java | 23 +++ .../jdk/incubator/vector/Byte512Vector.java | 23 +++ .../jdk/incubator/vector/Byte64Vector.java | 23 +++ .../jdk/incubator/vector/ByteMaxVector.java | 23 +++ .../jdk/incubator/vector/Double128Vector.java | 23 +++ .../jdk/incubator/vector/Double256Vector.java | 23 +++ .../jdk/incubator/vector/Double512Vector.java | 23 +++ .../jdk/incubator/vector/Double64Vector.java | 23 +++ .../jdk/incubator/vector/DoubleMaxVector.java | 23 +++ .../jdk/incubator/vector/Float128Vector.java | 23 +++ .../jdk/incubator/vector/Float256Vector.java | 23 +++ .../jdk/incubator/vector/Float512Vector.java | 23 +++ .../jdk/incubator/vector/Float64Vector.java | 23 +++ .../jdk/incubator/vector/FloatMaxVector.java | 23 +++ .../jdk/incubator/vector/Int128Vector.java | 23 +++ .../jdk/incubator/vector/Int256Vector.java | 23 +++ .../jdk/incubator/vector/Int512Vector.java | 23 +++ .../jdk/incubator/vector/Int64Vector.java | 23 +++ .../jdk/incubator/vector/IntMaxVector.java | 23 +++ .../jdk/incubator/vector/Long128Vector.java | 23 +++ .../jdk/incubator/vector/Long256Vector.java | 23 +++ .../jdk/incubator/vector/Long512Vector.java | 23 +++ .../jdk/incubator/vector/Long64Vector.java | 23 +++ .../jdk/incubator/vector/LongMaxVector.java | 23 +++ .../jdk/incubator/vector/Short128Vector.java | 23 +++ .../jdk/incubator/vector/Short256Vector.java | 23 +++ .../jdk/incubator/vector/Short512Vector.java | 23 +++ .../jdk/incubator/vector/Short64Vector.java | 23 +++ .../jdk/incubator/vector/ShortMaxVector.java | 23 +++ .../vector/X-VectorBits.java.template | 23 +++ .../incubator/vector/Byte128VectorTests.java | 18 ++ .../incubator/vector/Byte256VectorTests.java | 18 ++ .../incubator/vector/Byte512VectorTests.java | 18 ++ .../incubator/vector/Byte64VectorTests.java | 18 ++ .../incubator/vector/ByteMaxVectorTests.java | 18 ++ .../vector/Double128VectorTests.java | 18 ++ .../vector/Double256VectorTests.java | 18 ++ .../vector/Double512VectorTests.java | 18 ++ .../incubator/vector/Double64VectorTests.java | 18 ++ .../vector/DoubleMaxVectorTests.java | 18 ++ .../incubator/vector/Float128VectorTests.java | 18 ++ .../incubator/vector/Float256VectorTests.java | 18 ++ .../incubator/vector/Float512VectorTests.java | 18 ++ .../incubator/vector/Float64VectorTests.java | 18 ++ .../incubator/vector/FloatMaxVectorTests.java | 18 ++ .../incubator/vector/Int128VectorTests.java | 18 ++ .../incubator/vector/Int256VectorTests.java | 18 ++ .../incubator/vector/Int512VectorTests.java | 18 ++ .../incubator/vector/Int64VectorTests.java | 18 ++ .../incubator/vector/IntMaxVectorTests.java | 18 ++ .../incubator/vector/Long128VectorTests.java | 18 ++ .../incubator/vector/Long256VectorTests.java | 18 ++ .../incubator/vector/Long512VectorTests.java | 18 ++ .../incubator/vector/Long64VectorTests.java | 18 ++ .../incubator/vector/LongMaxVectorTests.java | 18 ++ .../incubator/vector/Short128VectorTests.java | 18 ++ .../incubator/vector/Short256VectorTests.java | 18 ++ .../incubator/vector/Short512VectorTests.java | 18 ++ .../incubator/vector/Short64VectorTests.java | 18 ++ .../incubator/vector/ShortMaxVectorTests.java | 18 ++ .../templates/Unit-Miscellaneous.template | 18 ++ .../vector/MaskQueryOperationsBenchmark.java | 159 ++++++++++++++++++ 81 files changed, 1821 insertions(+), 32 deletions(-) create mode 100644 mask.incr create mode 100644 test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java diff --git a/mask.incr b/mask.incr new file mode 100644 index 00000000000..d95a591a9fb --- /dev/null +++ b/mask.incr @@ -0,0 +1,43 @@ +diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp +index 3b8261d91d0..ad4bfd57f53 100644 +--- a/src/hotspot/share/opto/vectorIntrinsics.cpp ++++ b/src/hotspot/share/opto/vectorIntrinsics.cpp +@@ -429,8 +429,12 @@ bool LibraryCallKit::inline_vector_mask_operation() { + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + BasicType elem_bt = elem_type->basic_type(); + +- if (num_elem <= 2) { +- return false; ++ if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) { ++ if (C->print_intrinsics()) { ++ tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", ++ Op_LoadVector, num_elem, type2name(T_BOOLEAN)); ++ } ++ return false; // not supported + } + + int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt); +@@ -439,7 +443,7 @@ bool LibraryCallKit::inline_vector_mask_operation() { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", + mopc, num_elem, type2name(elem_bt)); + } +- return false; ++ return false; // not supported + } + + const Type* elem_ty = Type::get_const_basic_type(elem_bt); +diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java +index 50add676c62..6f9df1800f6 100644 +--- a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java ++++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java +@@ -84,8 +84,8 @@ public class MaskQueryOperationsBenchmark { + + @Setup(Level.Trial) + public void BmSetup() { +- bspecies = VectorSpecies.of(int.class, VectorShape.forBitSize(bits)); +- sspecies = VectorSpecies.of(int.class, VectorShape.forBitSize(bits)); ++ bspecies = VectorSpecies.of(byte.class, VectorShape.forBitSize(bits)); ++ sspecies = VectorSpecies.of(short.class, VectorShape.forBitSize(bits)); + ispecies = VectorSpecies.of(int.class, VectorShape.forBitSize(bits)); + lspecies = VectorSpecies.of(long.class, VectorShape.forBitSize(bits)); + diff --git a/src/hotspot/cpu/x86/assembler_x86.cpp b/src/hotspot/cpu/x86/assembler_x86.cpp index c46a65835ec..44a59b12765 100644 --- a/src/hotspot/cpu/x86/assembler_x86.cpp +++ b/src/hotspot/cpu/x86/assembler_x86.cpp @@ -9493,6 +9493,14 @@ void Assembler::shlxq(Register dst, Register src1, Register src2) { emit_int8((unsigned char)(0xC0 | encode)); } +void Assembler::evpmovb2m(KRegister dst, XMMRegister src, int vector_len) { + assert(VM_Version::supports_avx512vlbw(), ""); + InstructionAttr attributes(vector_len, /* vex_w */ false, /* legacy_mode */ false, /* no_mask_reg */ true, /* uses_vl */ true); + attributes.set_is_evex_instruction(); + int encode = vex_prefix_and_encode(dst->encoding(), 0, src->encoding(), VEX_SIMD_F3, VEX_OPCODE_0F_38, &attributes); + emit_int16(0x29, (0xC0 | encode)); +} + #ifndef _LP64 void Assembler::incl(Register dst) { diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp index 58a6ca7c9eb..7b37f982db9 100644 --- a/src/hotspot/cpu/x86/assembler_x86.hpp +++ b/src/hotspot/cpu/x86/assembler_x86.hpp @@ -2474,6 +2474,8 @@ class Assembler : public AbstractAssembler { void evpcmpw(KRegister kdst, KRegister mask, XMMRegister nds, Address src, int comparison, int vector_len); + void evpmovb2m(KRegister dst, XMMRegister src, int vector_len); + // Vector blends void blendvps(XMMRegister dst, XMMRegister src); void blendvpd(XMMRegister dst, XMMRegister src); diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 9deb87a523a..599fb6c17a6 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -1305,5 +1305,56 @@ void C2_MacroAssembler::vectortest(int bt, int vlen, XMMRegister src1, XMMRegist } } +#ifdef _LP64 +void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, + Register tmp, KRegister ktmp, int masklen, int vec_enc) { + assert(VM_Version::supports_avx512vlbw(), ""); + vpxor(xtmp, xtmp, xtmp, vec_enc); + vpsubb(xtmp, xtmp, mask, vec_enc); + evpmovb2m(ktmp, xtmp, vec_enc); + kmovql(tmp, ktmp); + switch(opc) { + case Op_VectorMaskTrueCount: + popcntq(dst, tmp); + break; + case Op_VectorMaskLastTrue: + mov64(dst, -1); + bsrq(tmp, tmp); + cmov(Assembler::notZero, dst, tmp); + break; + case Op_VectorMaskFirstTrue: + mov64(dst, masklen); + bsfq(tmp, tmp); + cmov(Assembler::notZero, dst, tmp); + break; + default: assert(false, "Unhandled mask operation"); + } +} + +void C2_MacroAssembler::vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, + XMMRegister xtmp1, Register tmp, int masklen, int vec_enc) { + assert(VM_Version::supports_avx(), ""); + vpxor(xtmp, xtmp, xtmp, vec_enc); + vpsubb(xtmp, xtmp, mask, vec_enc); + vpmovmskb(tmp, xtmp); + switch(opc) { + case Op_VectorMaskTrueCount: + popcntq(dst, tmp); + break; + case Op_VectorMaskLastTrue: + mov64(dst, -1); + bsrq(tmp, tmp); + cmov(Assembler::notZero, dst, tmp); + break; + case Op_VectorMaskFirstTrue: + mov64(dst, masklen); + bsfq(tmp, tmp); + cmov(Assembler::notZero, dst, tmp); + break; + default: assert(false, "Unhandled mask operation"); + } +} +#endif + #endif // COMPILER2 diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index ab574fb3716..eabaf78aa7f 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -156,4 +156,14 @@ // Base reduction instruction void reduce_operation_128(BasicType typ, int opcode, XMMRegister dst, XMMRegister src); void reduce_operation_256(BasicType typ, int opcode, XMMRegister dst, XMMRegister src1, XMMRegister src2); + + public: +#ifdef _LP64 + void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, Register tmp, + KRegister ktmp, int masklen, int vec_enc); + + void vector_mask_operation(int opc, Register dst, XMMRegister mask, XMMRegister xtmp, XMMRegister xtmp1, + Register tmp, int masklen, int vec_enc); +#endif + #endif // CPU_X86_C2_MACROASSEMBLER_X86_HPP diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 6909fb9c9d8..6d3b96cf136 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1681,6 +1681,13 @@ const bool Matcher::match_rule_supported(int opcode) { return false; } break; + case Op_VectorMaskFirstTrue: + case Op_VectorMaskLastTrue: + case Op_VectorMaskTrueCount: + if (!is_LP64 || UseAVX < 1) { + return false; + } + break; case Op_SqrtF: if (UseSSE < 1) { return false; @@ -8064,4 +8071,66 @@ instruct vmasked_store64(memory mem, vec src, kReg mask) %{ %} ins_pipe( pipe_slow ); %} + +instruct vmask_truecount_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp, vec xtmp) %{ + predicate(VM_Version::supports_avx512vlbw()); + match(Set dst (VectorMaskTrueCount mask)); + effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp, TEMP xtmp); + format %{ "vector_truecount_evex $mask \t! vector mask true count" %} + ins_encode %{ + int opcode = this->ideal_Opcode(); + int vlen_enc = vector_length_encoding(this, $mask); + int mask_len = vector_length(this, $mask); + __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister, + $tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc); + %} + ins_pipe( pipe_slow ); +%} + +instruct vmask_first_or_last_true_evex(rRegI dst, vec mask, rRegL tmp, kReg ktmp, vec xtmp, rFlagsReg cr) %{ + predicate(VM_Version::supports_avx512vlbw()); + match(Set dst (VectorMaskFirstTrue mask)); + match(Set dst (VectorMaskLastTrue mask)); + effect(TEMP_DEF dst, TEMP tmp, TEMP ktmp, TEMP xtmp, KILL cr); + format %{ "vector_mask_first_or_last_true_evex $mask \t! vector first/last true location" %} + ins_encode %{ + int opcode = this->ideal_Opcode(); + int vlen_enc = vector_length_encoding(this, $mask); + int mask_len = vector_length(this, $mask); + __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister, + $tmp$$Register, $ktmp$$KRegister, mask_len, vlen_enc); + %} + ins_pipe( pipe_slow ); +%} + +instruct vmask_truecount_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1) %{ + predicate(!VM_Version::supports_avx512vlbw()); + match(Set dst (VectorMaskTrueCount mask)); + effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, TEMP xtmp1); + format %{ "vector_truecount_avx $mask \t! vector mask true count" %} + ins_encode %{ + int opcode = this->ideal_Opcode(); + int vlen_enc = vector_length_encoding(this, $mask); + int mask_len = vector_length(this, $mask); + __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister, + $xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc); + %} + ins_pipe( pipe_slow ); +%} + +instruct vmask_first_or_last_true_avx(rRegI dst, vec mask, rRegL tmp, vec xtmp, vec xtmp1, rFlagsReg cr) %{ + predicate(!VM_Version::supports_avx512vlbw()); + match(Set dst (VectorMaskFirstTrue mask)); + match(Set dst (VectorMaskLastTrue mask)); + effect(TEMP_DEF dst, TEMP tmp, TEMP xtmp, TEMP xtmp1, KILL cr); + format %{ "vector_mask_first_or_last_true_avx $mask \t! vector first/last true location" %} + ins_encode %{ + int opcode = this->ideal_Opcode(); + int vlen_enc = vector_length_encoding(this, $mask); + int mask_len = vector_length(this, $mask); + __ vector_mask_operation(opcode, $dst$$Register, $mask$$XMMRegister, $xtmp$$XMMRegister, + $xtmp1$$XMMRegister, $tmp$$Register, mask_len, vlen_enc); + %} + ins_pipe( pipe_slow ); +%} #endif // _LP64 diff --git a/src/hotspot/share/classfile/vmSymbols.hpp b/src/hotspot/share/classfile/vmSymbols.hpp index 32cb352c3b7..5cced471ecb 100644 --- a/src/hotspot/share/classfile/vmSymbols.hpp +++ b/src/hotspot/share/classfile/vmSymbols.hpp @@ -1600,6 +1600,11 @@ do_alias(vector_rebox_sig, object_object_signature) \ do_name(vector_rebox_name, "maybeRebox") \ \ + do_intrinsic(_VectorMaskOp, jdk_internal_vm_vector_VectorSupport, vector_mask_oper_name, vector_mask_oper_sig, F_S) \ + do_signature(vector_mask_oper_sig, "(ILjava/lang/Class;Ljava/lang/Class;ILjava/lang/Object;" \ + "Ljdk/internal/vm/vector/VectorSupport$VectorMaskOp;)I") \ + do_name(vector_mask_oper_name, "maskReductionCoerced") \ + \ \ /* (2) Bytecode intrinsics */ \ \ @@ -1795,7 +1800,7 @@ class vmIntrinsics: AllStatic { #undef VM_INTRINSIC_ENUM ID_LIMIT, - LAST_COMPILER_INLINE = _VectorScatterOp, + LAST_COMPILER_INLINE = _VectorMaskOp, FIRST_MH_SIG_POLY = _invokeGeneric, FIRST_MH_STATIC = _linkToVirtual, LAST_MH_SIG_POLY = _linkToInterface, diff --git a/src/hotspot/share/opto/c2compiler.cpp b/src/hotspot/share/opto/c2compiler.cpp index 67cbc582919..e9a6131b445 100644 --- a/src/hotspot/share/opto/c2compiler.cpp +++ b/src/hotspot/share/opto/c2compiler.cpp @@ -679,6 +679,7 @@ bool C2Compiler::is_intrinsic_supported(const methodHandle& method, bool is_virt case vmIntrinsics::_VectorConvert: case vmIntrinsics::_VectorInsert: case vmIntrinsics::_VectorExtract: + case vmIntrinsics::_VectorMaskOp: return EnableVectorSupport; default: diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 47182f66d70..74982b5b777 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -397,6 +397,10 @@ macro(StoreVectorScatter) macro(LoadVectorMasked) macro(StoreVectorMasked) macro(VectorMaskGen) +macro(VectorMaskOp) +macro(VectorMaskTrueCount) +macro(VectorMaskFirstTrue) +macro(VectorMaskLastTrue) macro(Pack) macro(PackB) macro(PackS) diff --git a/src/hotspot/share/opto/library_call.cpp b/src/hotspot/share/opto/library_call.cpp index 5fef879945c..ebea8346e44 100644 --- a/src/hotspot/share/opto/library_call.cpp +++ b/src/hotspot/share/opto/library_call.cpp @@ -627,6 +627,8 @@ bool LibraryCallKit::try_to_inline(int predicate) { return inline_vector_broadcast_coerced(); case vmIntrinsics::_VectorShuffleIota: return inline_vector_shuffle_iota(); + case vmIntrinsics::_VectorMaskOp: + return inline_vector_mask_operation(); case vmIntrinsics::_VectorShuffleToVector: return inline_vector_shuffle_to_vector(); case vmIntrinsics::_VectorLoadOp: diff --git a/src/hotspot/share/opto/library_call.hpp b/src/hotspot/share/opto/library_call.hpp index 5950f0aecd9..511b8cf066e 100644 --- a/src/hotspot/share/opto/library_call.hpp +++ b/src/hotspot/share/opto/library_call.hpp @@ -310,6 +310,7 @@ class LibraryCallKit : public GraphKit { bool inline_vector_broadcast_coerced(); bool inline_vector_shuffle_to_vector(); bool inline_vector_shuffle_iota(); + bool inline_vector_mask_operation(); bool inline_vector_mem_operation(bool is_store); bool inline_vector_gather_scatter(bool is_scatter); bool inline_vector_reduction(); diff --git a/src/hotspot/share/opto/vectorIntrinsics.cpp b/src/hotspot/share/opto/vectorIntrinsics.cpp index 731e121723a..4a0e213a31d 100644 --- a/src/hotspot/share/opto/vectorIntrinsics.cpp +++ b/src/hotspot/share/opto/vectorIntrinsics.cpp @@ -408,6 +408,60 @@ bool LibraryCallKit::inline_vector_shuffle_iota() { return true; } +// +// int maskReductionCoerced(int oper, Class maskClass, Class elemClass, +// int length, M m, VectorMaskOp defaultImpl) +bool LibraryCallKit::inline_vector_mask_operation() { + const TypeInt* oper = gvn().type(argument(0))->isa_int(); + const TypeInstPtr* mask_klass = gvn().type(argument(1))->isa_instptr(); + const TypeInstPtr* elem_klass = gvn().type(argument(2))->isa_instptr(); + const TypeInt* vlen = gvn().type(argument(3))->isa_int(); + Node* mask = argument(4); + + if (mask_klass == NULL || elem_klass == NULL || mask->is_top() || vlen == NULL) { + return false; // dead code + } + + if (!is_klass_initialized(mask_klass)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** klass argument not initialized"); + } + return false; + } + + int num_elem = vlen->get_con(); + ciType* elem_type = elem_klass->const_oop()->as_instance()->java_mirror_type(); + BasicType elem_bt = elem_type->basic_type(); + + if (!arch_supports_vector(Op_LoadVector, num_elem, T_BOOLEAN, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", + Op_LoadVector, num_elem, type2name(T_BOOLEAN)); + } + return false; // not supported + } + + int mopc = VectorSupport::vop2ideal(oper->get_con(), elem_bt); + if (!arch_supports_vector(mopc, num_elem, elem_bt, VecMaskNotUsed)) { + if (C->print_intrinsics()) { + tty->print_cr(" ** not supported: arity=1 op=cast#%d/3 vlen2=%d etype2=%s", + mopc, num_elem, type2name(elem_bt)); + } + return false; // not supported + } + + const Type* elem_ty = Type::get_const_basic_type(elem_bt); + ciKlass* mbox_klass = mask_klass->const_oop()->as_instance()->java_lang_Class_klass(); + const TypeInstPtr* mask_box_type = TypeInstPtr::make_exact(TypePtr::NotNull, mbox_klass); + Node* mask_vec = unbox_vector(mask, mask_box_type, elem_bt, num_elem, true); + Node* store_mask = gvn().transform(VectorStoreMaskNode::make(gvn(), mask_vec, elem_bt, num_elem)); + Node* maskoper = gvn().transform(VectorMaskOpNode::make(store_mask, TypeInt::INT, mopc)); + set_result(maskoper); + + C->set_max_vector_size(MAX2(C->max_vector_size(), (uint)(num_elem * type2aelembytes(elem_bt)))); + return true; +} + // , E> // VM shuffleToVector(Class VecClass, ClassE , Class ShuffleClass, Sh s, int length, // ShuffleToVectorOperation defaultImpl) diff --git a/src/hotspot/share/opto/vectornode.cpp b/src/hotspot/share/opto/vectornode.cpp index 9975f2ab6e0..812ff7aebe4 100644 --- a/src/hotspot/share/opto/vectornode.cpp +++ b/src/hotspot/share/opto/vectornode.cpp @@ -1144,6 +1144,21 @@ const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) { return TypeFunc::make(domain, range); } +Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) { + switch(mopc) { + case Op_VectorMaskTrueCount: + return new VectorMaskTrueCountNode(mask, ty); + case Op_VectorMaskLastTrue: + return new VectorMaskLastTrueNode(mask, ty); + case Op_VectorMaskFirstTrue: + return new VectorMaskFirstTrueNode(mask, ty); + default: + assert(false, "Unhandled operation"); + } + return NULL; +} + + #ifndef PRODUCT void VectorBoxAllocateNode::dump_spec(outputStream *st) const { CallStaticJavaNode::dump_spec(st); diff --git a/src/hotspot/share/opto/vectornode.hpp b/src/hotspot/share/opto/vectornode.hpp index 4362bf7f4e5..60f3fa69dad 100644 --- a/src/hotspot/share/opto/vectornode.hpp +++ b/src/hotspot/share/opto/vectornode.hpp @@ -828,6 +828,45 @@ class VectorMaskGenNode : public TypeNode { const Type* _elemType; }; +class VectorMaskOpNode : public TypeNode { + public: + VectorMaskOpNode(Node* mask, const Type* ty, int mopc): + TypeNode(ty, 2), _mopc(mopc) { + assert(mask->Opcode() == Op_VectorStoreMask, ""); + init_req(1, mask); + } + + virtual int Opcode() const; + virtual uint size_of() const { return sizeof(VectorMaskOpNode); } + virtual uint ideal_reg() const { return Op_RegI; } + int get_mask_Opcode() const { return _mopc;} + static Node* make(Node* mask, const Type* ty, int mopc); + + private: + int _mopc; +}; + +class VectorMaskTrueCountNode : public VectorMaskOpNode { + public: + VectorMaskTrueCountNode(Node* mask, const Type* ty): + VectorMaskOpNode(mask, ty, Op_VectorMaskTrueCount) {} + virtual int Opcode() const; +}; + +class VectorMaskFirstTrueNode : public VectorMaskOpNode { + public: + VectorMaskFirstTrueNode(Node* mask, const Type* ty): + VectorMaskOpNode(mask, ty, Op_VectorMaskFirstTrue) {} + virtual int Opcode() const; +}; + +class VectorMaskLastTrueNode : public VectorMaskOpNode { + public: + VectorMaskLastTrueNode(Node* mask, const Type* ty): + VectorMaskOpNode(mask, ty, Op_VectorMaskLastTrue) {} + virtual int Opcode() const; +}; + //=========================Promote_Scalar_to_Vector============================ //------------------------------ReplicateBNode--------------------------------- diff --git a/src/hotspot/share/prims/vectorSupport.cpp b/src/hotspot/share/prims/vectorSupport.cpp index b0031afcdf8..ce89ca1f572 100644 --- a/src/hotspot/share/prims/vectorSupport.cpp +++ b/src/hotspot/share/prims/vectorSupport.cpp @@ -346,6 +346,42 @@ int VectorSupport::vop2ideal(jint id, BasicType bt) { } break; } + case VECTOR_OP_MASK_LASTTRUE: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: // fall-through + case T_LONG: // fall-through + case T_FLOAT: // fall-through + case T_DOUBLE: return Op_VectorMaskLastTrue; + default: fatal("MASK_LASTTRUE: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_MASK_FIRSTTRUE: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: // fall-through + case T_LONG: // fall-through + case T_FLOAT: // fall-through + case T_DOUBLE: return Op_VectorMaskFirstTrue; + default: fatal("MASK_FIRSTTRUE: %s", type2name(bt)); + } + break; + } + case VECTOR_OP_MASK_TRUECOUNT: { + switch (bt) { + case T_BYTE: // fall-through + case T_SHORT: // fall-through + case T_INT: // fall-through + case T_LONG: // fall-through + case T_FLOAT: // fall-through + case T_DOUBLE: return Op_VectorMaskTrueCount; + default: fatal("MASK_TRUECOUNT: %s", type2name(bt)); + } + break; + } default: fatal("unknown op: %d", vop); } return 0; // Unimplemented diff --git a/src/hotspot/share/prims/vectorSupport.hpp b/src/hotspot/share/prims/vectorSupport.hpp index 5debf65ac5b..27baf8feb51 100644 --- a/src/hotspot/share/prims/vectorSupport.hpp +++ b/src/hotspot/share/prims/vectorSupport.hpp @@ -77,7 +77,12 @@ class VectorSupport : AllStatic { // Convert VECTOR_OP_CAST = 17, - VECTOR_OP_REINTERPRET = 18 + VECTOR_OP_REINTERPRET = 18, + + // Mask manipulation operations + VECTOR_OP_MASK_TRUECOUNT = 19, + VECTOR_OP_MASK_FIRSTTRUE = 20, + VECTOR_OP_MASK_LASTTRUE = 21 }; static int vop2ideal(jint vop, BasicType bt); diff --git a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java index 2171b20f033..7f02c9accce 100644 --- a/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java +++ b/src/java.base/share/classes/jdk/internal/vm/vector/VectorSupport.java @@ -69,6 +69,11 @@ public class VectorSupport { public static final int VECTOR_OP_CAST = 17; public static final int VECTOR_OP_REINTERPRET = 18; + // Mask manipulation operations + public static final int VECTOR_OP_MASK_TRUECOUNT = 19; + public static final int VECTOR_OP_MASK_FIRSTTRUE = 20; + public static final int VECTOR_OP_MASK_LASTTRUE = 21; + // enum BoolTest public static final int BT_eq = 0; public static final int BT_ne = 4; @@ -453,6 +458,20 @@ public static V maybeRebox(V v) { return v; } + /* ============================================================================ */ + public interface VectorMaskOp { + int apply(M m); + } + + @HotSpotIntrinsicCandidate + public static + + int maskReductionCoerced(int oper, Class maskClass, Class elemClass, int length, M m, + VectorMaskOp defaultImpl) { + assert isNonCapturingLambda(defaultImpl) : defaultImpl; + return defaultImpl.apply(m); + } + /* ============================================================================ */ // query the JVM's supported vector sizes and types diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java index f7b75f199f1..5169bad0d72 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/AbstractMask.java @@ -114,36 +114,6 @@ VectorMask check(VectorSpecies species) { return (VectorMask) this; } - @Override - public int trueCount() { - //FIXME: use a population count intrinsic here - int c = 0; - for (boolean i : getBits()) { - if (i) c++; - } - return c; - } - - @Override - public int firstTrue() { - //FIXME: use a count trailing zeros intrinsic here - boolean[] bits = getBits(); - for (int i = 0; i < bits.length; i++) { - if (bits[i]) return i; - } - return bits.length; - } - - @Override - public int lastTrue() { - //FIXME: use a count leading zeros intrinsic here - boolean[] bits = getBits(); - for (int i = bits.length-1; i >= 0; i--) { - if (bits[i]) return i; - } - return -1; - } - @Override public VectorMask eq(VectorMask m) { // FIXME: Generate good code here. @@ -173,6 +143,31 @@ static boolean allTrueHelper(boolean[] bits) { return true; } + /*package-private*/ + static int trueCountHelper(boolean[] bits) { + int c = 0; + for (boolean i : bits) { + if (i) c++; + } + return c; + } + + /*package-private*/ + static int firstTrueHelper(boolean[] bits) { + for (int i = 0; i < bits.length; i++) { + if (bits[i]) return i; + } + return bits.length; + } + + /*package-private*/ + static int lastTrueHelper(boolean[] bits) { + for (int i = bits.length-1; i >= 0; i--) { + if (bits[i]) return i; + } + return -1; + } + @Override @ForceInline public VectorMask indexInRange(int offset, int limit) { diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java index e2baacff09d..fce28d1fda3 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java @@ -666,6 +666,29 @@ Byte128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Byte128Mask.class, byte.class, VLENGTH, this, + (m) -> trueCountHelper(((Byte128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Byte128Mask.class, byte.class, VLENGTH, this, + (m) -> firstTrueHelper(((Byte128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Byte128Mask.class, byte.class, VLENGTH, this, + (m) -> lastTrueHelper(((Byte128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java index f100935e6f0..2c687ef5b89 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java @@ -698,6 +698,29 @@ Byte256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Byte256Mask.class, byte.class, VLENGTH, this, + (m) -> trueCountHelper(((Byte256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Byte256Mask.class, byte.class, VLENGTH, this, + (m) -> firstTrueHelper(((Byte256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Byte256Mask.class, byte.class, VLENGTH, this, + (m) -> lastTrueHelper(((Byte256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java index 70a23a22fd1..ce2eae9425c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java @@ -762,6 +762,29 @@ Byte512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Byte512Mask.class, byte.class, VLENGTH, this, + (m) -> trueCountHelper(((Byte512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Byte512Mask.class, byte.class, VLENGTH, this, + (m) -> firstTrueHelper(((Byte512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Byte512Mask.class, byte.class, VLENGTH, this, + (m) -> lastTrueHelper(((Byte512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java index e33be0fb12c..057da8e5478 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte64Vector.java @@ -650,6 +650,29 @@ Byte64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Byte64Mask.class, byte.class, VLENGTH, this, + (m) -> trueCountHelper(((Byte64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Byte64Mask.class, byte.class, VLENGTH, this, + (m) -> firstTrueHelper(((Byte64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Byte64Mask.class, byte.class, VLENGTH, this, + (m) -> lastTrueHelper(((Byte64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java index 19ec4bf57e6..7978a30ea18 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ByteMaxVector.java @@ -636,6 +636,29 @@ ByteMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, ByteMaxMask.class, byte.class, VLENGTH, this, + (m) -> trueCountHelper(((ByteMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, ByteMaxMask.class, byte.class, VLENGTH, this, + (m) -> firstTrueHelper(((ByteMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, ByteMaxMask.class, byte.class, VLENGTH, this, + (m) -> lastTrueHelper(((ByteMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java index 6df75dec501..b5849d2cb4d 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double128Vector.java @@ -634,6 +634,29 @@ Double128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Double128Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Double128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Double128Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Double128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Double128Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Double128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java index d4058fb25f7..bf524f021df 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double256Vector.java @@ -638,6 +638,29 @@ Double256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Double256Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Double256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Double256Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Double256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Double256Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Double256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java index 04f27791871..94d4db1510e 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double512Vector.java @@ -646,6 +646,29 @@ Double512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Double512Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Double512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Double512Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Double512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Double512Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Double512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java index 14672601009..640f0acd25c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Double64Vector.java @@ -632,6 +632,29 @@ Double64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Double64Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Double64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Double64Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Double64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Double64Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Double64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java index 54b5a9f4805..392a796a4a0 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/DoubleMaxVector.java @@ -631,6 +631,29 @@ DoubleMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, DoubleMaxMask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((DoubleMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, DoubleMaxMask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((DoubleMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, DoubleMaxMask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((DoubleMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java index a1a9634f4e9..18a755d0116 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float128Vector.java @@ -638,6 +638,29 @@ Float128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Float128Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Float128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Float128Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Float128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Float128Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Float128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java index 69786cffc4c..6083b754d96 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float256Vector.java @@ -646,6 +646,29 @@ Float256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Float256Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Float256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Float256Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Float256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Float256Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Float256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java index 41b9222fffb..b9e233b1250 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float512Vector.java @@ -662,6 +662,29 @@ Float512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Float512Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Float512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Float512Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Float512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Float512Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Float512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java index 0c615a9bc9d..02a45a6c183 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Float64Vector.java @@ -634,6 +634,29 @@ Float64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Float64Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Float64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Float64Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Float64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Float64Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Float64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java index 1a6f6c6c7ea..84a7ec7ddbe 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/FloatMaxVector.java @@ -631,6 +631,29 @@ FloatMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, FloatMaxMask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((FloatMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, FloatMaxMask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((FloatMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, FloatMaxMask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((FloatMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java index d5d39fc36d9..7cf81f6c6a6 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int128Vector.java @@ -642,6 +642,29 @@ Int128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Int128Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Int128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Int128Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Int128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Int128Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Int128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java index 15f2361ebcd..17f2f1d4b4b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int256Vector.java @@ -650,6 +650,29 @@ Int256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Int256Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Int256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Int256Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Int256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Int256Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Int256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java index 1d6551e5379..3c50e941d1f 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int512Vector.java @@ -666,6 +666,29 @@ Int512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Int512Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Int512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Int512Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Int512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Int512Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Int512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java index d5ef529f11d..8c83c3f5350 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Int64Vector.java @@ -638,6 +638,29 @@ Int64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Int64Mask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((Int64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Int64Mask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((Int64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Int64Mask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((Int64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java index bb75b346db8..4904ff27dca 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/IntMaxVector.java @@ -636,6 +636,29 @@ IntMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, IntMaxMask.class, int.class, VLENGTH, this, + (m) -> trueCountHelper(((IntMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, IntMaxMask.class, int.class, VLENGTH, this, + (m) -> firstTrueHelper(((IntMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, IntMaxMask.class, int.class, VLENGTH, this, + (m) -> lastTrueHelper(((IntMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java index a3801a317c4..89a6fc666c8 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long128Vector.java @@ -628,6 +628,29 @@ Long128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Long128Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Long128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Long128Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Long128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Long128Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Long128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java index 7f4526c0b33..f1d07749924 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long256Vector.java @@ -632,6 +632,29 @@ Long256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Long256Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Long256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Long256Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Long256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Long256Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Long256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java index b940bd6f290..38905798b8b 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long512Vector.java @@ -640,6 +640,29 @@ Long512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Long512Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Long512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Long512Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Long512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Long512Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Long512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java index b38985b7a5f..c61a13f834c 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Long64Vector.java @@ -626,6 +626,29 @@ Long64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Long64Mask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((Long64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Long64Mask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((Long64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Long64Mask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((Long64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java index 04b627e1c6a..502c183eb02 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/LongMaxVector.java @@ -626,6 +626,29 @@ LongMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, LongMaxMask.class, long.class, VLENGTH, this, + (m) -> trueCountHelper(((LongMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, LongMaxMask.class, long.class, VLENGTH, this, + (m) -> firstTrueHelper(((LongMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, LongMaxMask.class, long.class, VLENGTH, this, + (m) -> lastTrueHelper(((LongMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java index a1c702394ad..60c3985de88 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short128Vector.java @@ -650,6 +650,29 @@ Short128Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Short128Mask.class, short.class, VLENGTH, this, + (m) -> trueCountHelper(((Short128Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Short128Mask.class, short.class, VLENGTH, this, + (m) -> firstTrueHelper(((Short128Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Short128Mask.class, short.class, VLENGTH, this, + (m) -> lastTrueHelper(((Short128Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java index 1d552f35df4..16cab237e77 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short256Vector.java @@ -666,6 +666,29 @@ Short256Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Short256Mask.class, short.class, VLENGTH, this, + (m) -> trueCountHelper(((Short256Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Short256Mask.class, short.class, VLENGTH, this, + (m) -> firstTrueHelper(((Short256Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Short256Mask.class, short.class, VLENGTH, this, + (m) -> lastTrueHelper(((Short256Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java index ccd6b8ccf62..bfc74a349fb 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short512Vector.java @@ -698,6 +698,29 @@ Short512Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Short512Mask.class, short.class, VLENGTH, this, + (m) -> trueCountHelper(((Short512Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Short512Mask.class, short.class, VLENGTH, this, + (m) -> firstTrueHelper(((Short512Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Short512Mask.class, short.class, VLENGTH, this, + (m) -> lastTrueHelper(((Short512Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java index c8fa39031c6..1da076aaa03 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Short64Vector.java @@ -642,6 +642,29 @@ Short64Mask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, Short64Mask.class, short.class, VLENGTH, this, + (m) -> trueCountHelper(((Short64Mask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, Short64Mask.class, short.class, VLENGTH, this, + (m) -> firstTrueHelper(((Short64Mask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, Short64Mask.class, short.class, VLENGTH, this, + (m) -> lastTrueHelper(((Short64Mask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java index 302fa04a7bc..8de1b464277 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/ShortMaxVector.java @@ -636,6 +636,29 @@ ShortMaxMask xor(VectorMask mask) { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, ShortMaxMask.class, short.class, VLENGTH, this, + (m) -> trueCountHelper(((ShortMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, ShortMaxMask.class, short.class, VLENGTH, this, + (m) -> firstTrueHelper(((ShortMaxMask)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, ShortMaxMask.class, short.class, VLENGTH, this, + (m) -> lastTrueHelper(((ShortMaxMask)m).getBits())); + } + // Reductions @Override diff --git a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template index 634960e5167..8303b28b933 100644 --- a/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template +++ b/src/jdk.incubator.vector/share/classes/jdk/incubator/vector/X-VectorBits.java.template @@ -909,6 +909,29 @@ final class $vectortype$ extends $abstractvectortype$ { (m1, m2) -> m1.bOp(m2, (i, a, b) -> a ^ b)); } + // Mask Query operations + + @Override + @ForceInline + public int trueCount() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_TRUECOUNT, $masktype$.class, $bitstype$.class, VLENGTH, this, + (m) -> trueCountHelper((($masktype$)m).getBits())); + } + + @Override + @ForceInline + public int firstTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_FIRSTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this, + (m) -> firstTrueHelper((($masktype$)m).getBits())); + } + + @Override + @ForceInline + public int lastTrue() { + return VectorSupport.maskReductionCoerced(VECTOR_OP_MASK_LASTTRUE, $masktype$.class, $bitstype$.class, VLENGTH, this, + (m) -> lastTrueHelper((($masktype$)m).getBits())); + } + // Reductions @Override diff --git a/test/jdk/jdk/incubator/vector/Byte128VectorTests.java b/test/jdk/jdk/incubator/vector/Byte128VectorTests.java index a7d006daec3..a3861e75f3c 100644 --- a/test/jdk/jdk/incubator/vector/Byte128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Byte128VectorTests.java @@ -5178,6 +5178,24 @@ static void maskLastTrueByte128VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueByte128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Byte256VectorTests.java b/test/jdk/jdk/incubator/vector/Byte256VectorTests.java index 7b68b9a4fef..1d80c929cd9 100644 --- a/test/jdk/jdk/incubator/vector/Byte256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Byte256VectorTests.java @@ -5178,6 +5178,24 @@ static void maskLastTrueByte256VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueByte256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Byte512VectorTests.java b/test/jdk/jdk/incubator/vector/Byte512VectorTests.java index 61e4aa5008d..d0371c79cf9 100644 --- a/test/jdk/jdk/incubator/vector/Byte512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Byte512VectorTests.java @@ -5178,6 +5178,24 @@ static void maskLastTrueByte512VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueByte512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Byte64VectorTests.java b/test/jdk/jdk/incubator/vector/Byte64VectorTests.java index affeea486e4..5f810647bbe 100644 --- a/test/jdk/jdk/incubator/vector/Byte64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Byte64VectorTests.java @@ -5178,6 +5178,24 @@ static void maskLastTrueByte64VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueByte64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/ByteMaxVectorTests.java b/test/jdk/jdk/incubator/vector/ByteMaxVectorTests.java index d6dd82af7ae..72005730caa 100644 --- a/test/jdk/jdk/incubator/vector/ByteMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/ByteMaxVectorTests.java @@ -5183,6 +5183,24 @@ static void maskLastTrueByteMaxVectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueByteMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/Double128VectorTests.java b/test/jdk/jdk/incubator/vector/Double128VectorTests.java index b1faaf7ced1..11ec69a87c8 100644 --- a/test/jdk/jdk/incubator/vector/Double128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Double128VectorTests.java @@ -4924,6 +4924,24 @@ static void maskLastTrueDouble128VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueDouble128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Double256VectorTests.java b/test/jdk/jdk/incubator/vector/Double256VectorTests.java index 1e1a1a851f5..3d30815992b 100644 --- a/test/jdk/jdk/incubator/vector/Double256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Double256VectorTests.java @@ -4924,6 +4924,24 @@ static void maskLastTrueDouble256VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueDouble256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Double512VectorTests.java b/test/jdk/jdk/incubator/vector/Double512VectorTests.java index 86975ef14ee..488898f9bce 100644 --- a/test/jdk/jdk/incubator/vector/Double512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Double512VectorTests.java @@ -4924,6 +4924,24 @@ static void maskLastTrueDouble512VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueDouble512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Double64VectorTests.java b/test/jdk/jdk/incubator/vector/Double64VectorTests.java index a4c21b5ff98..2c648659a54 100644 --- a/test/jdk/jdk/incubator/vector/Double64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Double64VectorTests.java @@ -4924,6 +4924,24 @@ static void maskLastTrueDouble64VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueDouble64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/DoubleMaxVectorTests.java b/test/jdk/jdk/incubator/vector/DoubleMaxVectorTests.java index dbe35d51b22..f5a736e1d49 100644 --- a/test/jdk/jdk/incubator/vector/DoubleMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/DoubleMaxVectorTests.java @@ -4929,6 +4929,24 @@ static void maskLastTrueDoubleMaxVectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueDoubleMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/Float128VectorTests.java b/test/jdk/jdk/incubator/vector/Float128VectorTests.java index 3f2e01c0343..da2cddbcdec 100644 --- a/test/jdk/jdk/incubator/vector/Float128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Float128VectorTests.java @@ -4902,6 +4902,24 @@ static void maskLastTrueFloat128VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueFloat128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Float256VectorTests.java b/test/jdk/jdk/incubator/vector/Float256VectorTests.java index 40a561c5260..3e5be0ecd5a 100644 --- a/test/jdk/jdk/incubator/vector/Float256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Float256VectorTests.java @@ -4902,6 +4902,24 @@ static void maskLastTrueFloat256VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueFloat256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Float512VectorTests.java b/test/jdk/jdk/incubator/vector/Float512VectorTests.java index 91556d5d9df..2d9eebf3de0 100644 --- a/test/jdk/jdk/incubator/vector/Float512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Float512VectorTests.java @@ -4902,6 +4902,24 @@ static void maskLastTrueFloat512VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueFloat512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Float64VectorTests.java b/test/jdk/jdk/incubator/vector/Float64VectorTests.java index 9e00b5bfc36..8e567f57891 100644 --- a/test/jdk/jdk/incubator/vector/Float64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Float64VectorTests.java @@ -4902,6 +4902,24 @@ static void maskLastTrueFloat64VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueFloat64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/FloatMaxVectorTests.java b/test/jdk/jdk/incubator/vector/FloatMaxVectorTests.java index 842c4f93075..1997645ee80 100644 --- a/test/jdk/jdk/incubator/vector/FloatMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/FloatMaxVectorTests.java @@ -4907,6 +4907,24 @@ static void maskLastTrueFloatMaxVectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueFloatMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/Int128VectorTests.java b/test/jdk/jdk/incubator/vector/Int128VectorTests.java index 5139c08801d..9ad6a2b806b 100644 --- a/test/jdk/jdk/incubator/vector/Int128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Int128VectorTests.java @@ -5133,6 +5133,24 @@ static void maskLastTrueInt128VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueInt128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Int256VectorTests.java b/test/jdk/jdk/incubator/vector/Int256VectorTests.java index 37e44fa3f3d..d79d1ce49d9 100644 --- a/test/jdk/jdk/incubator/vector/Int256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Int256VectorTests.java @@ -5133,6 +5133,24 @@ static void maskLastTrueInt256VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueInt256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Int512VectorTests.java b/test/jdk/jdk/incubator/vector/Int512VectorTests.java index 943cb444ec5..dd73991c6d0 100644 --- a/test/jdk/jdk/incubator/vector/Int512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Int512VectorTests.java @@ -5133,6 +5133,24 @@ static void maskLastTrueInt512VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueInt512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Int64VectorTests.java b/test/jdk/jdk/incubator/vector/Int64VectorTests.java index 9a3a9541423..2bde75eeea0 100644 --- a/test/jdk/jdk/incubator/vector/Int64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Int64VectorTests.java @@ -5133,6 +5133,24 @@ static void maskLastTrueInt64VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueInt64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/IntMaxVectorTests.java b/test/jdk/jdk/incubator/vector/IntMaxVectorTests.java index 1db67b4b2de..54728ef130d 100644 --- a/test/jdk/jdk/incubator/vector/IntMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/IntMaxVectorTests.java @@ -5138,6 +5138,24 @@ static void maskLastTrueIntMaxVectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueIntMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/Long128VectorTests.java b/test/jdk/jdk/incubator/vector/Long128VectorTests.java index 19ce3554e88..984e2ff7991 100644 --- a/test/jdk/jdk/incubator/vector/Long128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Long128VectorTests.java @@ -5016,6 +5016,24 @@ static void maskLastTrueLong128VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueLong128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Long256VectorTests.java b/test/jdk/jdk/incubator/vector/Long256VectorTests.java index 6bd614f5a7c..f541341d41a 100644 --- a/test/jdk/jdk/incubator/vector/Long256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Long256VectorTests.java @@ -5016,6 +5016,24 @@ static void maskLastTrueLong256VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueLong256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Long512VectorTests.java b/test/jdk/jdk/incubator/vector/Long512VectorTests.java index ca02ac5b726..25151061f92 100644 --- a/test/jdk/jdk/incubator/vector/Long512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Long512VectorTests.java @@ -5016,6 +5016,24 @@ static void maskLastTrueLong512VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueLong512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Long64VectorTests.java b/test/jdk/jdk/incubator/vector/Long64VectorTests.java index 2ba810968c0..c1310fcf917 100644 --- a/test/jdk/jdk/incubator/vector/Long64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Long64VectorTests.java @@ -5016,6 +5016,24 @@ static void maskLastTrueLong64VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueLong64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/LongMaxVectorTests.java b/test/jdk/jdk/incubator/vector/LongMaxVectorTests.java index 364ae80ddc5..cb6679b9070 100644 --- a/test/jdk/jdk/incubator/vector/LongMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/LongMaxVectorTests.java @@ -5021,6 +5021,24 @@ static void maskLastTrueLongMaxVectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueLongMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/Short128VectorTests.java b/test/jdk/jdk/incubator/vector/Short128VectorTests.java index 91f3e215759..97ae371a730 100644 --- a/test/jdk/jdk/incubator/vector/Short128VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Short128VectorTests.java @@ -5157,6 +5157,24 @@ static void maskLastTrueShort128VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueShort128VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Short256VectorTests.java b/test/jdk/jdk/incubator/vector/Short256VectorTests.java index efbde094120..0a3da47a9f1 100644 --- a/test/jdk/jdk/incubator/vector/Short256VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Short256VectorTests.java @@ -5157,6 +5157,24 @@ static void maskLastTrueShort256VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueShort256VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Short512VectorTests.java b/test/jdk/jdk/incubator/vector/Short512VectorTests.java index 16463ba45d8..7d46134f9a1 100644 --- a/test/jdk/jdk/incubator/vector/Short512VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Short512VectorTests.java @@ -5157,6 +5157,24 @@ static void maskLastTrueShort512VectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueShort512VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/Short64VectorTests.java b/test/jdk/jdk/incubator/vector/Short64VectorTests.java index adc10b3b4b9..c4972418fba 100644 --- a/test/jdk/jdk/incubator/vector/Short64VectorTests.java +++ b/test/jdk/jdk/incubator/vector/Short64VectorTests.java @@ -5157,6 +5157,24 @@ static void maskLastTrueShort64VectorTestsSmokeTest(IntFunction fa) { } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueShort64VectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] longMaskProvider() { return new Object[][]{ diff --git a/test/jdk/jdk/incubator/vector/ShortMaxVectorTests.java b/test/jdk/jdk/incubator/vector/ShortMaxVectorTests.java index 7e4d7e3fa09..603afb6a8ae 100644 --- a/test/jdk/jdk/incubator/vector/ShortMaxVectorTests.java +++ b/test/jdk/jdk/incubator/vector/ShortMaxVectorTests.java @@ -5162,6 +5162,24 @@ static void maskLastTrueShortMaxVectorTestsSmokeTest(IntFunction fa) } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrueShortMaxVectorTestsSmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + @DataProvider public static Object[][] offsetProvider() { diff --git a/test/jdk/jdk/incubator/vector/templates/Unit-Miscellaneous.template b/test/jdk/jdk/incubator/vector/templates/Unit-Miscellaneous.template index 6e97d1517d7..93ca2065752 100644 --- a/test/jdk/jdk/incubator/vector/templates/Unit-Miscellaneous.template +++ b/test/jdk/jdk/incubator/vector/templates/Unit-Miscellaneous.template @@ -444,6 +444,24 @@ } } + @Test(dataProvider = "maskProvider") + static void maskFirstTrue$vectorteststype$SmokeTest(IntFunction fa) { + boolean[] a = fa.apply(SPECIES.length()); + + for (int i = 0; i < a.length; i += SPECIES.length()) { + var vmask = SPECIES.loadMask(a, i); + int ftrue = vmask.firstTrue(); + int j = i; + for (; j < i + SPECIES.length() ; j++) { + if (a[j]) break; + } + int expectedFtrue = j - i; + + Assert.assertTrue(ftrue == expectedFtrue, "at index " + i + + ", firstTrue should be = " + expectedFtrue + ", but is = " + ftrue); + } + } + #if[!MaxBit] @DataProvider public static Object[][] longMaskProvider() { diff --git a/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java new file mode 100644 index 00000000000..6f9df1800f6 --- /dev/null +++ b/test/micro/org/openjdk/bench/jdk/incubator/vector/MaskQueryOperationsBenchmark.java @@ -0,0 +1,159 @@ +// +// Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved. +// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +// +// This code is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License version 2 only, as +// published by the Free Software Foundation. +// +// This code is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +// version 2 for more details (a copy is included in the LICENSE file that +// accompanied this code). +// +// You should have received a copy of the GNU General Public License version +// 2 along with this work; if not, write to the Free Software Foundation, +// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +// +// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +// or visit www.oracle.com if you need additional information or have any +// questions. +// +// +package org.openjdk.bench.jdk.incubator.vector; + +import java.util.Random; +import jdk.incubator.vector.*; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Thread) +public class MaskQueryOperationsBenchmark { + @Param({"128","256","512"}) + int bits; + + @Param({"1","2","3"}) + int inputs; + + VectorSpecies bspecies; + VectorSpecies sspecies; + VectorSpecies ispecies; + VectorSpecies lspecies; + VectorMask bmask; + VectorMask smask; + VectorMask imask; + VectorMask lmask; + boolean [] mask_arr; + + + static final boolean [] mask_avg_case = { + false, false, false, true, false, false, false, false, + false, false, false, true, false, false, false, false, + false, false, false, true, false, false, false, false, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + false, false, false, true, false, false, false, false, + false, false, false, true, false, false, false, false, + false, false, false, true, false, false, false, false + }; + + static final boolean [] mask_best_case = { + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true, + true, true, true, true, true, true, true, true + }; + + static final boolean [] mask_worst_case = { + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false + }; + + @Setup(Level.Trial) + public void BmSetup() { + bspecies = VectorSpecies.of(byte.class, VectorShape.forBitSize(bits)); + sspecies = VectorSpecies.of(short.class, VectorShape.forBitSize(bits)); + ispecies = VectorSpecies.of(int.class, VectorShape.forBitSize(bits)); + lspecies = VectorSpecies.of(long.class, VectorShape.forBitSize(bits)); + + if( 1 == inputs) { + mask_arr = mask_best_case; + } else if ( 2 == inputs ) { + mask_arr = mask_worst_case; + } else { + mask_arr = mask_avg_case; + } + + bmask = VectorMask.fromArray(bspecies, mask_arr, 0); + smask = VectorMask.fromArray(sspecies, mask_arr, 0); + imask = VectorMask.fromArray(ispecies, mask_arr, 0); + lmask = VectorMask.fromArray(lspecies, mask_arr, 0); + } + + @Benchmark + public int testTrueCountByte(Blackhole bh) { + return bmask.trueCount(); + } + + @Benchmark + public int testTrueCountShort(Blackhole bh) { + return smask.trueCount(); + } + @Benchmark + public int testTrueCountInt(Blackhole bh) { + return imask.trueCount(); + } + @Benchmark + public int testTrueCountLong(Blackhole bh) { + return lmask.trueCount(); + } + + @Benchmark + public int testFirstTrueByte(Blackhole bh) { + return bmask.firstTrue(); + } + + @Benchmark + public int testFirstTrueShort(Blackhole bh) { + return smask.firstTrue(); + } + @Benchmark + public int testFirstTrueInt(Blackhole bh) { + return imask.firstTrue(); + } + @Benchmark + public int testFirstTrueLong(Blackhole bh) { + return lmask.firstTrue(); + } + + @Benchmark + public int testLastTrueByte(Blackhole bh) { + return bmask.lastTrue(); + } + + @Benchmark + public int testLastTrueShort(Blackhole bh) { + return smask.lastTrue(); + } + @Benchmark + public int testLastTrueInt(Blackhole bh) { + return imask.lastTrue(); + } + @Benchmark + public int testLastTrueLong(Blackhole bh) { + return lmask.lastTrue(); + } +}