Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Backport] 8305898: Alternative self-forwarding mechanism
Browse files Browse the repository at this point in the history
Summary: Alternative self-forwarding mechanism under flag UseAltGCForwarding.
         Now support G1 and parallelGC.
         Add missing gtest of UseAltGCForwarding from 8305896.

Test Plan: CICD

Reviewed-by: yude.lin, yifeng.jin

Issue: #692
mmyxym committed Oct 23, 2023
1 parent d9a0bfb commit 94e6965
Showing 10 changed files with 206 additions and 6 deletions.
2 changes: 1 addition & 1 deletion src/hotspot/share/gc/g1/g1OopClosures.inline.hpp
Original file line number Diff line number Diff line change
@@ -251,7 +251,7 @@ void G1ParCopyClosure<barrier, do_mark_object>::do_oop_work(T* p) {
oop forwardee;
markOop m = obj->mark_raw();
if (m->is_marked()) {
forwardee = (oop) m->decode_pointer();
forwardee = UseAltGCForwarding ? obj->forwardee(m) : (oop) m->decode_pointer();
} else {
forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m);
}
3 changes: 2 additions & 1 deletion src/hotspot/share/gc/g1/g1ParScanThreadState.cpp
Original file line number Diff line number Diff line change
@@ -362,7 +362,8 @@ void G1ParScanThreadStateSet::flush() {
oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markOop m) {
assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old));

oop forward_ptr = old->forward_to_atomic(old, memory_order_relaxed);
oop forward_ptr = UseAltGCForwarding ? old->forward_to_self_atomic(m, memory_order_relaxed) :
old->forward_to_atomic(old, memory_order_relaxed);
if (forward_ptr == NULL) {
// Forward-to-self succeeded. We are the "owner" of the object.
HeapRegion* r = _g1h->heap_region_containing(old);
2 changes: 1 addition & 1 deletion src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ template <class T> void G1ParScanThreadState::do_oop_evac(T* p) {
if (in_cset_state.is_in_cset()) {
markOop m = obj->mark_raw();
if (m->is_marked()) {
obj = (oop) m->decode_pointer();
obj = UseAltGCForwarding ? obj->forwardee(m) : (oop) m->decode_pointer();
} else {
obj = copy_to_survivor_space(in_cset_state, obj, m);
}
2 changes: 1 addition & 1 deletion src/hotspot/share/gc/parallel/psPromotionManager.cpp
Original file line number Diff line number Diff line change
@@ -500,7 +500,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) {
// this started. If it is the same (i.e., no forwarding
// pointer has been installed), then this thread owns
// it.
if (obj->cas_forward_to(obj, obj_mark)) {
if (UseAltGCForwarding ? (obj->forward_to_self_atomic(obj_mark) == NULL) : obj->cas_forward_to(obj, obj_mark)) {
// We won any races, we "own" this object.
assert(obj == obj->forwardee(), "Sanity");

16 changes: 16 additions & 0 deletions src/hotspot/share/oops/markOop.hpp
Original file line number Diff line number Diff line change
@@ -131,6 +131,9 @@ class markOopDesc: public oopDesc {
biased_lock_mask = right_n_bits(lock_bits + biased_lock_bits),
biased_lock_mask_in_place= biased_lock_mask << lock_shift,
biased_lock_bit_in_place = 1 << biased_lock_shift,
// self forward bit has the same position of biased lock bit
self_forwarded_mask = right_n_bits(biased_lock_bits),
self_forwarded_mask_in_place = self_forwarded_mask << biased_lock_shift,
age_mask = right_n_bits(age_bits),
age_mask_in_place = age_mask << age_shift,
epoch_mask = right_n_bits(epoch_bits),
@@ -371,6 +374,19 @@ class markOopDesc: public oopDesc {
// Recover address of oop from encoded form used in mark
inline void* decode_pointer() { if (UseBiasedLocking && has_bias_pattern()) return NULL; return clear_lock_bits(); }

#ifdef _LP64
inline bool self_forwarded() const {
bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0;
assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding");
return self_fwd;
}

inline markOop set_self_forwarded() const {
assert(UseAltGCForwarding, "Only call this with alt GC forwarding");
return markOop(value() | self_forwarded_mask_in_place | marked_value);
}
#endif

// These markOops indicate cms free chunk blocks and not objects.
// In 64 bit, the markOop is set to distinguish them from oops.
// These are defined in 32 bit mode for vmStructs.
3 changes: 3 additions & 0 deletions src/hotspot/share/oops/oop.hpp
Original file line number Diff line number Diff line change
@@ -268,10 +268,13 @@ class oopDesc {
// this call returns "NULL" for that thread; any other thread has the
// value of the forwarding pointer returned and does not modify "this".
inline oop forward_to_atomic(oop p, atomic_memory_order order = memory_order_conservative);
inline oop forward_to_self_atomic(markOop m, atomic_memory_order order = memory_order_conservative);

inline oop forwardee() const;
inline oop forwardee_acquire() const;

inline oop forwardee(markOop header) const;

// Age of object during scavenge
inline uint age() const;
inline void incr_age();
48 changes: 46 additions & 2 deletions src/hotspot/share/oops/oop.inline.hpp
Original file line number Diff line number Diff line change
@@ -392,19 +392,63 @@ oop oopDesc::forward_to_atomic(oop p, atomic_memory_order order) {
return forwardee();
}

oop oopDesc::forward_to_self_atomic(markOop compare, atomic_memory_order order) {
#ifdef _LP64
assert(UseAltGCForwarding, "sanity");
markOop m = compare;
// If mark is displaced, we need to preserve the real header during GC.
// It will be restored to the displaced header after GC.
assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint");
if (m->has_displaced_mark_helper()) {
m = m->displaced_mark_helper();
}
m = m->set_self_forwarded();
assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible");
markOop old_mark = cas_set_mark_raw(m, compare, order);
if (old_mark == compare) {
return NULL;
} else {
assert(old_mark->is_marked(), "must be marked here");
return forwardee(old_mark);
}
#else
return forward_to_atomic(cast_to_oop(this), order);
#endif
}

// Note that the forwardee is not the same thing as the displaced_mark.
// The forwardee is used when copying during scavenge and mark-sweep.
// It does need to clear the low two locking- and GC-related bits.
oop oopDesc::forwardee() const {
return (oop) mark_raw()->decode_pointer();
if (UseAltGCForwarding) {
return forwardee(mark());
} else {
return (oop) mark_raw()->decode_pointer();
}
}

oop oopDesc::forwardee(markOop header) const {
assert(header->is_marked(), "only decode when actually forwarded");
#ifdef _LP64
if (header->self_forwarded()) {
return cast_to_oop(this);
} else
#endif
{
return cast_to_oop(header->decode_pointer());
}
}

// Note that the forwardee is not the same thing as the displaced_mark.
// The forwardee is used when copying during scavenge and mark-sweep.
// It does need to clear the low two locking- and GC-related bits.
oop oopDesc::forwardee_acquire() const {
markOop m = OrderAccess::load_acquire(&_mark);
return (oop) m->decode_pointer();
if (UseAltGCForwarding) {
return forwardee(m);
} else {
return (oop) m->decode_pointer();
}
}

// The following method needs to be MT safe.
7 changes: 7 additions & 0 deletions src/hotspot/share/runtime/arguments.cpp
Original file line number Diff line number Diff line change
@@ -4179,6 +4179,13 @@ jint Arguments::apply_ergo() {
UseBiasedLocking = false;
}

if (UseAltGCForwarding) {
if (!(UseG1GC || (UseParallelGC && UseParallelOldGC))) {
warning("UseAltGCForwarding is supported with current GC setting.");
FLAG_SET_DEFAULT(UseAltGCForwarding, false);
}
}

#ifdef CC_INTERP
// Clear flags not supported on zero.
FLAG_SET_DEFAULT(ProfileInterpreter, false);
2 changes: 2 additions & 0 deletions test/hotspot/gtest/gc/shared/test_preservedMarks.cpp
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@ TEST_VM(PreservedMarks, iterate_and_restore) {
FakeOop o3;
FakeOop o4;

FlagSetting fs(UseAltGCForwarding, false);

// Make sure initial marks are correct.
ASSERT_EQ(o1.mark(), FakeOop::originalMark());
ASSERT_EQ(o2.mark(), FakeOop::originalMark());
127 changes: 127 additions & 0 deletions test/hotspot/gtest/gc/shared/test_slidingForwarding.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/

#include "precompiled.hpp"
#include "gc/shared/gc_globals.hpp"
#include "gc/shared/slidingForwarding.inline.hpp"
#include "oops/markOop.hpp"
#include "oops/oop.inline.hpp"
#include "utilities/align.hpp"
#include "unittest.hpp"

#ifdef _LP64
#ifndef PRODUCT

static uintptr_t make_mark(uintptr_t target_region, uintptr_t offset) {
return (target_region) << 3 | (offset << 4) | 3 /* forwarded */;
}

static uintptr_t make_fallback() {
return ((uintptr_t(1) << 2) /* fallback */ | 3 /* forwarded */);
}

// Test simple forwarding within the same region.
TEST_VM(SlidingForwarding, simple) {
FlagSetting fs(UseAltGCForwarding, true);

HeapWord* fakeheap[32] = { NULL };
HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord));
oop obj1 = cast_to_oop(&heap[2]);
oop obj2 = cast_to_oop(&heap[0]);
SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8);
obj1->set_mark(markOopDesc::prototype());
SlidingForwarding::begin();

SlidingForwarding::forward_to<true>(obj1, obj2);
ASSERT_EQ(obj1->mark()->value(), make_mark(0 /* target_region */, 0 /* offset */));
ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj2);

SlidingForwarding::end();
}

// Test forwardings crossing 2 regions.
TEST_VM(SlidingForwarding, tworegions) {
FlagSetting fs(UseAltGCForwarding, true);

HeapWord* fakeheap[32] = { NULL };
HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord));
oop obj1 = cast_to_oop(&heap[14]);
oop obj2 = cast_to_oop(&heap[2]);
oop obj3 = cast_to_oop(&heap[10]);
SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8);
obj1->set_mark(markOopDesc::prototype());
SlidingForwarding::begin();

SlidingForwarding::forward_to<true>(obj1, obj2);
ASSERT_EQ(obj1->mark()->value(), make_mark(0 /* target_region */, 2 /* offset */));
ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj2);

SlidingForwarding::forward_to<true>(obj1, obj3);
ASSERT_EQ(obj1->mark()->value(), make_mark(1 /* target_region */, 2 /* offset */));
ASSERT_EQ(SlidingForwarding::forwardee<true>(obj1), obj3);

SlidingForwarding::end();
}

// Test fallback forwardings crossing 4 regions.
TEST_VM(SlidingForwarding, fallback) {
FlagSetting fs(UseAltGCForwarding, true);

HeapWord* fakeheap[32] = { NULL };
HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord));
oop s_obj1 = cast_to_oop(&heap[12]);
oop s_obj2 = cast_to_oop(&heap[13]);
oop s_obj3 = cast_to_oop(&heap[14]);
oop s_obj4 = cast_to_oop(&heap[15]);
oop t_obj1 = cast_to_oop(&heap[2]);
oop t_obj2 = cast_to_oop(&heap[4]);
oop t_obj3 = cast_to_oop(&heap[10]);
oop t_obj4 = cast_to_oop(&heap[12]);
SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 4);
s_obj1->set_mark(markOopDesc::prototype());
s_obj2->set_mark(markOopDesc::prototype());
s_obj3->set_mark(markOopDesc::prototype());
s_obj4->set_mark(markOopDesc::prototype());
SlidingForwarding::begin();

SlidingForwarding::forward_to<true>(s_obj1, t_obj1);
ASSERT_EQ(s_obj1->mark()->value(), make_mark(0 /* target_region */, 2 /* offset */));
ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj1), t_obj1);

SlidingForwarding::forward_to<true>(s_obj2, t_obj2);
ASSERT_EQ(s_obj2->mark()->value(), make_mark(1 /* target_region */, 0 /* offset */));
ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj2), t_obj2);

SlidingForwarding::forward_to<true>(s_obj3, t_obj3);
ASSERT_EQ(s_obj3->mark()->value(), make_fallback());
ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj3), t_obj3);

SlidingForwarding::forward_to<true>(s_obj4, t_obj4);
ASSERT_EQ(s_obj4->mark()->value(), make_fallback());
ASSERT_EQ(SlidingForwarding::forwardee<true>(s_obj4), t_obj4);

SlidingForwarding::end();
}

#endif // PRODUCT
#endif // _LP64

0 comments on commit 94e6965

Please sign in to comment.