[Backport] JDK-8264759, JDK-8264104, JDK-8265816, JDK-8266180, JDK-8265128, JDK-8265126

src/hotspot/cpu/x86/sharedRuntime_x86_32.cpp

@@ -217,6 +217,7 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_
     }
   }
 
+#ifdef COMPILER2
   if (save_vectors) {
     __ subptr(rsp, ymm_bytes);
     // Save upper half of YMM registers
@@ -236,6 +237,10 @@ OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_
       }
     }
   }
+#else
+  assert(!save_vectors, "vectors are generated only by C2");
+#endif
+
   __ vzeroupper();
 
   // Set an oopmap for the call site.  This oopmap will map all

src/hotspot/cpu/x86/x86.ad

@@ -7589,6 +7589,18 @@ instruct vstoreMask8B_evex(vec dst, vec src, immI_8 size) %{
   ins_pipe( pipe_slow );
 %}
 
+instruct vmaskcast(vec dst) %{
+  predicate((vector_length(n) == vector_length(n->in(1))) &&
+            (vector_length_in_bytes(n) == vector_length_in_bytes(n->in(1))));
+  match(Set dst (VectorMaskCast dst));
+  ins_cost(0);
+  format %{ "vector_mask_cast $dst" %}
+  ins_encode %{
+    // empty
+  %}
+  ins_pipe(empty);
+%}
+
 //-------------------------------- Load Iota Indices ----------------------------------
 
 instruct loadIotaIndices(vec dst, immI_0 src, rRegP scratch) %{

src/hotspot/share/adlc/formssel.cpp

@@ -4201,7 +4201,8 @@ bool MatchRule::is_vector() const {
     "FmaVD", "FmaVF","PopCountVI",
     // Next are not supported currently.
     "PackB","PackS","PackI","PackL","PackF","PackD","Pack2L","Pack2D",
-    "ExtractB","ExtractUB","ExtractC","ExtractS","ExtractI","ExtractL","ExtractF","ExtractD"
+    "ExtractB","ExtractUB","ExtractC","ExtractS","ExtractI","ExtractL","ExtractF","ExtractD",
+    "VectorMaskCast"
   };
   int cnt = sizeof(vector_list)/sizeof(char*);
   if (_rChild) {

src/hotspot/share/opto/classes.hpp

@@ -430,6 +430,7 @@ macro(VectorBoxAllocate)
 macro(VectorUnbox)
 macro(VectorMaskWrapper)
 macro(VectorMaskCmp)
+macro(VectorMaskCast)
 macro(VectorTest)
 macro(VectorBlend)
 macro(VectorRearrange)

src/hotspot/share/opto/vector.cpp

@@ -30,6 +30,15 @@
 #include "opto/vector.hpp"
 #include "utilities/macros.hpp"
 
+static bool is_vector_mask(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
+}
+
+static bool is_vector_shuffle(ciKlass* klass) {
+  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
+}
+
+
 void PhaseVector::optimize_vector_boxes() {
   Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
 
@@ -236,13 +245,24 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
     }
   }
 
+  ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass();
+  int n_fields = iklass->nof_nonstatic_fields();
+  assert(n_fields == 1, "sanity");
+
+  // If a mask is feeding into safepoint[s], then its value should be
+  // packed into a boolean/byte vector first, this will simplify the
+  // re-materialization logic for both predicated and non-predicated
+  // targets.
+  bool is_mask = is_vector_mask(iklass);
+  if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
+    const TypeVect* vt = vec_value->bottom_type()->is_vect();
+    BasicType bt = vt->element_basic_type();
+    vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
+  }
+
   while (safepoints.size() > 0) {
     SafePointNode* sfpt = safepoints.pop()->as_SafePoint();
 
-    ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass();
-    int n_fields = iklass->nof_nonstatic_fields();
-    assert(n_fields == 1, "sanity");
-
     uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
     Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(),
 #ifdef ASSERT
@@ -303,14 +323,6 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
   }
 }
 
-static bool is_vector_mask(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
-}
-
-static bool is_vector_shuffle(ciKlass* klass) {
-  return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
-}
-
 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
                                           Node* value,
                                           const TypeInstPtr* box_type,

src/hotspot/share/opto/vectornode.cpp

@@ -1095,6 +1095,13 @@ Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) {
       bool is_vector_mask    = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
       bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
       if (is_vector_mask) {
+        if (in_vt->length_in_bytes() == out_vt->length_in_bytes() &&
+            Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) {
+          // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)"
+          // directly. This could avoid the transformation ordering issue from
+          // "VectorStoreMask (VectorLoadMask vmask) => vmask".
+          return new VectorMaskCastNode(value, out_vt);
+        }
         // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask)
         value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length()));
         return new VectorLoadMaskNode(value, out_vt);

src/hotspot/share/opto/vectornode.hpp

@@ -1215,6 +1215,17 @@ class VectorStoreMaskNode : public VectorNode {
   static VectorStoreMaskNode* make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem);
 };
 
+class VectorMaskCastNode : public VectorNode {
+ public:
+  VectorMaskCastNode(Node* in, const TypeVect* vt) : VectorNode(in, vt) {
+    const TypeVect* in_vt = in->bottom_type()->is_vect();
+    assert(in_vt->length() == vt->length(), "vector length must match");
+    assert(type2aelembytes(in_vt->element_basic_type()) == type2aelembytes(vt->element_basic_type()), "element size must match");
+  }
+
+  virtual int Opcode() const;
+};
+
 // This is intended for use as a simple reinterpret node that has no cast.
 class VectorReinterpretNode : public VectorNode {
  private:

src/hotspot/share/prims/vectorSupport.cpp

@@ -62,6 +62,8 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
 
   if (is_vector_shuffle(ik)) {
     return T_BYTE;
+  } else if (is_vector_mask(ik)) {
+    return T_BOOLEAN;
   } else { // vector and mask
     oop value = ik->java_mirror()->obj_field(fd.offset());
     BasicType elem_bt = java_lang_Class::as_BasicType(value);
@@ -83,48 +85,33 @@ jint VectorSupport::klass2length(InstanceKlass* ik) {
   return vlen;
 }
 
-void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) {
-  if (is_mask) {
-    // Masks require special handling: when boxed they are packed and stored in boolean
-    // arrays, but in scalarized form they have the same size as corresponding vectors.
-    // For example, Int512Mask is represented in memory as boolean[16], but
-    // occupies the whole 512-bit vector register when scalarized.
-    // (In generated code, the conversion is performed by VectorStoreMask.)
-    //
-    // TODO: revisit when predicate registers are fully supported.
-    switch (elem_bt) {
-      case T_BYTE:   arr->bool_at_put(index,  (*(jbyte*)addr) != 0); break;
-      case T_SHORT:  arr->bool_at_put(index, (*(jshort*)addr) != 0); break;
-      case T_INT:    // fall-through
-      case T_FLOAT:  arr->bool_at_put(index,   (*(jint*)addr) != 0); break;
-      case T_LONG:   // fall-through
-      case T_DOUBLE: arr->bool_at_put(index,  (*(jlong*)addr) != 0); break;
-
-      default: fatal("unsupported: %s", type2name(elem_bt));
-    }
-  } else {
-    switch (elem_bt) {
-      case T_BYTE:   arr->  byte_at_put(index,   *(jbyte*)addr); break;
-      case T_SHORT:  arr-> short_at_put(index,  *(jshort*)addr); break;
-      case T_INT:    arr->   int_at_put(index,    *(jint*)addr); break;
-      case T_FLOAT:  arr-> float_at_put(index,  *(jfloat*)addr); break;
-      case T_LONG:   arr->  long_at_put(index,   *(jlong*)addr); break;
-      case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;
-
-      default: fatal("unsupported: %s", type2name(elem_bt));
-    }
+// Masks require special handling: when boxed they are packed and stored in boolean
+// arrays, but in scalarized form they have the same size as corresponding vectors.
+// For example, Int512Mask is represented in memory as boolean[16], but
+// occupies the whole 512-bit vector register when scalarized.
+// During scalarization inserting a VectorStoreMask node between mask
+// and safepoint node always ensures the existence of masks in a boolean array.
+
+void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) {
+  switch (elem_bt) {
+    case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break;
+    case T_BYTE:    arr->byte_at_put(index, *(jbyte*)addr); break;
+    case T_SHORT:   arr->short_at_put(index, *(jshort*)addr); break;
+    case T_INT:     arr->int_at_put(index, *(jint*)addr); break;
+    case T_FLOAT:   arr->float_at_put(index, *(jfloat*)addr); break;
+    case T_LONG:    arr->long_at_put(index, *(jlong*)addr); break;
+    case T_DOUBLE:  arr->double_at_put(index, *(jdouble*)addr); break;
+    default: fatal("unsupported: %s", type2name(elem_bt));
   }
 }
 
 Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) {
-  bool is_mask = is_vector_mask(ik);
-
   int num_elem = klass2length(ik);
   BasicType elem_bt = klass2bt(ik);
   int elem_size = type2aelembytes(elem_bt);
 
   // On-heap vector values are represented as primitive arrays.
-  TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt));
+  TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt));
 
   typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint
 
@@ -137,13 +124,13 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f
       int off   = (i * elem_size) % VMRegImpl::stack_slot_size;
 
       address elem_addr = reg_map->location(vreg->next(vslot)) + off;
-      init_payload_element(arr, is_mask, elem_bt, i, elem_addr);
+      init_payload_element(arr, elem_bt, i, elem_addr);
     }
   } else {
     // Value was directly saved on the stack.
     address base_addr = ((address)fr->unextended_sp()) + location.stack_offset();
     for (int i = 0; i < num_elem; i++) {
-      init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size);
+      init_payload_element(arr, elem_bt, i, base_addr + i * elem_size);
     }
   }
   return Handle(THREAD, arr);

src/hotspot/share/prims/vectorSupport.hpp

@@ -42,7 +42,7 @@ class VectorSupport : AllStatic {
   static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS);
   static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS);
 
-  static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr);
+  static void init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr);
 
   static BasicType klass2bt(InstanceKlass* ik);
   static jint klass2length(InstanceKlass* ik);

src/hotspot/share/runtime/vmStructs.cpp

@@ -1924,6 +1924,7 @@ typedef PaddedEnd<ObjectMonitor>              PaddedObjectMonitor;
   declare_c2_type(VectorInsertNode, VectorNode)                           \
   declare_c2_type(VectorUnboxNode, VectorNode)                            \
   declare_c2_type(VectorReinterpretNode, VectorNode)                      \
+  declare_c2_type(VectorMaskCastNode, VectorNode)                         \
   declare_c2_type(VectorBoxNode, Node)                                    \
   declare_c2_type(VectorBoxAllocateNode, CallStaticJavaNode)              \
   declare_c2_type(VectorTestNode, Node)                                   \

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte128Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -387,14 +387,7 @@ public Byte128Vector slice(int origin, Vector<Byte> v) {
     @Override
     @ForceInline
     public Byte128Vector slice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte128Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin))));
-         Iota = iotaShuffle(origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte128Vector) super.sliceTemplate(origin);  // specialize
     }
 
     @Override
@@ -415,14 +408,7 @@ public Byte128Vector unslice(int origin, Vector<Byte> w, int part, VectorMask<By
     @Override
     @ForceInline
     public Byte128Vector unslice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte128Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin))));
-         Iota = iotaShuffle(-origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte128Vector) super.unsliceTemplate(origin);  // specialize
     }
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte256Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -387,14 +387,7 @@ public Byte256Vector slice(int origin, Vector<Byte> v) {
     @Override
     @ForceInline
     public Byte256Vector slice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte256Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin))));
-         Iota = iotaShuffle(origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte256Vector) super.sliceTemplate(origin);  // specialize
     }
 
     @Override
@@ -415,14 +408,7 @@ public Byte256Vector unslice(int origin, Vector<Byte> w, int part, VectorMask<By
     @Override
     @ForceInline
     public Byte256Vector unslice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte256Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin))));
-         Iota = iotaShuffle(-origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte256Vector) super.unsliceTemplate(origin);  // specialize
     }
 
     @Override

src/jdk.incubator.vector/share/classes/jdk/incubator/vector/Byte512Vector.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -387,14 +387,7 @@ public Byte512Vector slice(int origin, Vector<Byte> v) {
     @Override
     @ForceInline
     public Byte512Vector slice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte512Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.LT, (broadcast((byte)(VLENGTH-origin))));
-         Iota = iotaShuffle(origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte512Vector) super.sliceTemplate(origin);  // specialize
     }
 
     @Override
@@ -415,14 +408,7 @@ public Byte512Vector unslice(int origin, Vector<Byte> w, int part, VectorMask<By
     @Override
     @ForceInline
     public Byte512Vector unslice(int origin) {
-       if ((origin < 0) || (origin >= VLENGTH)) {
-         throw new ArrayIndexOutOfBoundsException("Index " + origin + " out of bounds for vector length " + VLENGTH);
-       } else {
-         Byte512Shuffle Iota = iotaShuffle();
-         VectorMask<Byte> BlendMask = Iota.toVector().compare(VectorOperators.GE, (broadcast((byte)(origin))));
-         Iota = iotaShuffle(-origin, 1, true);
-         return ZERO.blend(this.rearrange(Iota), BlendMask);
-       }
+        return (Byte512Vector) super.unsliceTemplate(origin);  // specialize
     }
 
     @Override