Skip to content

Commit

Permalink
[Backport] 8265126: [REDO] unified handling for VectorMask object re-…
Browse files Browse the repository at this point in the history 
…materialization during de-optimization

Summary: 8265126: [REDO] unified handling for VectorMask object re-materialization during de-optimization

Test Plan: ci jtreg

Reviewed-by: JoshuaZhuwj

Issue: #599
  • Loading branch information
@JinZhonghui @JoshuaZhuwj
JinZhonghui authored and JoshuaZhuwj committed Nov 6, 2023
1 parent 94a06e9 commit fa355c6
Show file tree
Hide file tree
Showing 3 changed files with 47 additions and 48 deletions.
36 changes: 24 additions & 12 deletions src/hotspot/share/opto/vector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,15 @@
#include "opto/vector.hpp"
#include "utilities/macros.hpp"

static bool is_vector_mask(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
}

static bool is_vector_shuffle(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
}


void PhaseVector::optimize_vector_boxes() {
Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);

Expand Down Expand Up @@ -236,13 +245,24 @@ void PhaseVector::scalarize_vbox_node(VectorBoxNode* vec_box) {
}
}

ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass();
int n_fields = iklass->nof_nonstatic_fields();
assert(n_fields == 1, "sanity");

// If a mask is feeding into safepoint[s], then its value should be
// packed into a boolean/byte vector first, this will simplify the
// re-materialization logic for both predicated and non-predicated
// targets.
bool is_mask = is_vector_mask(iklass);
if (is_mask && vec_value->Opcode() != Op_VectorStoreMask) {
const TypeVect* vt = vec_value->bottom_type()->is_vect();
BasicType bt = vt->element_basic_type();
vec_value = gvn.transform(VectorStoreMaskNode::make(gvn, vec_value, bt, vt->length()));
}

while (safepoints.size() > 0) {
SafePointNode* sfpt = safepoints.pop()->as_SafePoint();

ciInstanceKlass* iklass = vec_box->box_type()->klass()->as_instance_klass();
int n_fields = iklass->nof_nonstatic_fields();
assert(n_fields == 1, "sanity");

uint first_ind = (sfpt->req() - sfpt->jvms()->scloff());
Node* sobj = new SafePointScalarObjectNode(vec_box->box_type(),
#ifdef ASSERT
Expand Down Expand Up @@ -303,14 +323,6 @@ Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
}
}

static bool is_vector_mask(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
}

static bool is_vector_shuffle(ciKlass* klass) {
return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
}

Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
Node* value,
const TypeInstPtr* box_type,
Expand Down
57 changes: 22 additions & 35 deletions src/hotspot/share/prims/vectorSupport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ BasicType VectorSupport::klass2bt(InstanceKlass* ik) {

if (is_vector_shuffle(ik)) {
return T_BYTE;
} else if (is_vector_mask(ik)) {
return T_BOOLEAN;
} else { // vector and mask
oop value = ik->java_mirror()->obj_field(fd.offset());
BasicType elem_bt = java_lang_Class::as_BasicType(value);
Expand All @@ -83,48 +85,33 @@ jint VectorSupport::klass2length(InstanceKlass* ik) {
return vlen;
}

void VectorSupport::init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr) {
if (is_mask) {
// Masks require special handling: when boxed they are packed and stored in boolean
// arrays, but in scalarized form they have the same size as corresponding vectors.
// For example, Int512Mask is represented in memory as boolean[16], but
// occupies the whole 512-bit vector register when scalarized.
// (In generated code, the conversion is performed by VectorStoreMask.)
//
// TODO: revisit when predicate registers are fully supported.
switch (elem_bt) {
case T_BYTE: arr->bool_at_put(index, (*(jbyte*)addr) != 0); break;
case T_SHORT: arr->bool_at_put(index, (*(jshort*)addr) != 0); break;
case T_INT: // fall-through
case T_FLOAT: arr->bool_at_put(index, (*(jint*)addr) != 0); break;
case T_LONG: // fall-through
case T_DOUBLE: arr->bool_at_put(index, (*(jlong*)addr) != 0); break;

default: fatal("unsupported: %s", type2name(elem_bt));
}
} else {
switch (elem_bt) {
case T_BYTE: arr-> byte_at_put(index, *(jbyte*)addr); break;
case T_SHORT: arr-> short_at_put(index, *(jshort*)addr); break;
case T_INT: arr-> int_at_put(index, *(jint*)addr); break;
case T_FLOAT: arr-> float_at_put(index, *(jfloat*)addr); break;
case T_LONG: arr-> long_at_put(index, *(jlong*)addr); break;
case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;

default: fatal("unsupported: %s", type2name(elem_bt));
}
// Masks require special handling: when boxed they are packed and stored in boolean
// arrays, but in scalarized form they have the same size as corresponding vectors.
// For example, Int512Mask is represented in memory as boolean[16], but
// occupies the whole 512-bit vector register when scalarized.
// During scalarization inserting a VectorStoreMask node between mask
// and safepoint node always ensures the existence of masks in a boolean array.

void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) {
switch (elem_bt) {
case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break;
case T_BYTE: arr->byte_at_put(index, *(jbyte*)addr); break;
case T_SHORT: arr->short_at_put(index, *(jshort*)addr); break;
case T_INT: arr->int_at_put(index, *(jint*)addr); break;
case T_FLOAT: arr->float_at_put(index, *(jfloat*)addr); break;
case T_LONG: arr->long_at_put(index, *(jlong*)addr); break;
case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;
default: fatal("unsupported: %s", type2name(elem_bt));
}
}

Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) {
bool is_mask = is_vector_mask(ik);

int num_elem = klass2length(ik);
BasicType elem_bt = klass2bt(ik);
int elem_size = type2aelembytes(elem_bt);

// On-heap vector values are represented as primitive arrays.
TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(is_mask ? T_BOOLEAN : elem_bt));
TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt));

typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint

Expand All @@ -137,13 +124,13 @@ Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* f
int off = (i * elem_size) % VMRegImpl::stack_slot_size;

address elem_addr = reg_map->location(vreg->next(vslot)) + off;
init_payload_element(arr, is_mask, elem_bt, i, elem_addr);
init_payload_element(arr, elem_bt, i, elem_addr);
}
} else {
// Value was directly saved on the stack.
address base_addr = ((address)fr->unextended_sp()) + location.stack_offset();
for (int i = 0; i < num_elem; i++) {
init_payload_element(arr, is_mask, elem_bt, i, base_addr + i * elem_size);
init_payload_element(arr, elem_bt, i, base_addr + i * elem_size);
}
}
return Handle(THREAD, arr);
Expand Down
2 changes: 1 addition & 1 deletion src/hotspot/share/prims/vectorSupport.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ class VectorSupport : AllStatic {
static Handle allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS);
static Handle allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS);

static void init_payload_element(typeArrayOop arr, bool is_mask, BasicType elem_bt, int index, address addr);
static void init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr);

static BasicType klass2bt(InstanceKlass* ik);
static jint klass2length(InstanceKlass* ik);
Expand Down

0 comments on commit fa355c6

Please sign in to comment.