diff --git a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp index c2549132350..3a5682b2fbf 100644 --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp @@ -251,7 +251,7 @@ void G1ParCopyClosure::do_oop_work(T* p) { oop forwardee; markOop m = obj->mark_raw(); if (m->is_marked()) { - forwardee = (oop) m->decode_pointer(); + forwardee = UseAltGCForwarding ? obj->forwardee(m) : (oop) m->decode_pointer(); } else { forwardee = _par_scan_state->copy_to_survivor_space(state, obj, m); } diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp index 0c8089274a7..57c2517cc1d 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp @@ -362,7 +362,8 @@ void G1ParScanThreadStateSet::flush() { oop G1ParScanThreadState::handle_evacuation_failure_par(oop old, markOop m) { assert(_g1h->is_in_cset(old), "Object " PTR_FORMAT " should be in the CSet", p2i(old)); - oop forward_ptr = old->forward_to_atomic(old, memory_order_relaxed); + oop forward_ptr = UseAltGCForwarding ? old->forward_to_self_atomic(m, memory_order_relaxed) : + old->forward_to_atomic(old, memory_order_relaxed); if (forward_ptr == NULL) { // Forward-to-self succeeded. We are the "owner" of the object. HeapRegion* r = _g1h->heap_region_containing(old); diff --git a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp index 124d2ca87e5..e6d2fadd28a 100644 --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp @@ -43,7 +43,7 @@ template void G1ParScanThreadState::do_oop_evac(T* p) { if (in_cset_state.is_in_cset()) { markOop m = obj->mark_raw(); if (m->is_marked()) { - obj = (oop) m->decode_pointer(); + obj = UseAltGCForwarding ? obj->forwardee(m) : (oop) m->decode_pointer(); } else { obj = copy_to_survivor_space(in_cset_state, obj, m); } diff --git a/src/hotspot/share/gc/parallel/psPromotionManager.cpp b/src/hotspot/share/gc/parallel/psPromotionManager.cpp index b792486bc60..3db29a5ad41 100644 --- a/src/hotspot/share/gc/parallel/psPromotionManager.cpp +++ b/src/hotspot/share/gc/parallel/psPromotionManager.cpp @@ -500,7 +500,7 @@ oop PSPromotionManager::oop_promotion_failed(oop obj, markOop obj_mark) { // this started. If it is the same (i.e., no forwarding // pointer has been installed), then this thread owns // it. - if (obj->cas_forward_to(obj, obj_mark)) { + if (UseAltGCForwarding ? (obj->forward_to_self_atomic(obj_mark) == NULL) : obj->cas_forward_to(obj, obj_mark)) { // We won any races, we "own" this object. assert(obj == obj->forwardee(), "Sanity"); diff --git a/src/hotspot/share/oops/markOop.hpp b/src/hotspot/share/oops/markOop.hpp index be78d9907f5..12210bf0d5c 100644 --- a/src/hotspot/share/oops/markOop.hpp +++ b/src/hotspot/share/oops/markOop.hpp @@ -131,6 +131,9 @@ class markOopDesc: public oopDesc { biased_lock_mask = right_n_bits(lock_bits + biased_lock_bits), biased_lock_mask_in_place= biased_lock_mask << lock_shift, biased_lock_bit_in_place = 1 << biased_lock_shift, + // self forward bit has the same position of biased lock bit + self_forwarded_mask = right_n_bits(biased_lock_bits), + self_forwarded_mask_in_place = self_forwarded_mask << biased_lock_shift, age_mask = right_n_bits(age_bits), age_mask_in_place = age_mask << age_shift, epoch_mask = right_n_bits(epoch_bits), @@ -371,6 +374,19 @@ class markOopDesc: public oopDesc { // Recover address of oop from encoded form used in mark inline void* decode_pointer() { if (UseBiasedLocking && has_bias_pattern()) return NULL; return clear_lock_bits(); } +#ifdef _LP64 + inline bool self_forwarded() const { + bool self_fwd = mask_bits(value(), self_forwarded_mask_in_place) != 0; + assert(!self_fwd || UseAltGCForwarding, "Only set self-fwd bit when using alt GC forwarding"); + return self_fwd; + } + + inline markOop set_self_forwarded() const { + assert(UseAltGCForwarding, "Only call this with alt GC forwarding"); + return markOop(value() | self_forwarded_mask_in_place | marked_value); + } +#endif + // These markOops indicate cms free chunk blocks and not objects. // In 64 bit, the markOop is set to distinguish them from oops. // These are defined in 32 bit mode for vmStructs. diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp index 4430530e5e4..bb6a1dba5fb 100644 --- a/src/hotspot/share/oops/oop.hpp +++ b/src/hotspot/share/oops/oop.hpp @@ -268,10 +268,13 @@ class oopDesc { // this call returns "NULL" for that thread; any other thread has the // value of the forwarding pointer returned and does not modify "this". inline oop forward_to_atomic(oop p, atomic_memory_order order = memory_order_conservative); + inline oop forward_to_self_atomic(markOop m, atomic_memory_order order = memory_order_conservative); inline oop forwardee() const; inline oop forwardee_acquire() const; + inline oop forwardee(markOop header) const; + // Age of object during scavenge inline uint age() const; inline void incr_age(); diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp index 6c631f54584..8580dd5781c 100644 --- a/src/hotspot/share/oops/oop.inline.hpp +++ b/src/hotspot/share/oops/oop.inline.hpp @@ -392,11 +392,51 @@ oop oopDesc::forward_to_atomic(oop p, atomic_memory_order order) { return forwardee(); } +oop oopDesc::forward_to_self_atomic(markOop compare, atomic_memory_order order) { +#ifdef _LP64 + assert(UseAltGCForwarding, "sanity"); + markOop m = compare; + // If mark is displaced, we need to preserve the real header during GC. + // It will be restored to the displaced header after GC. + assert(SafepointSynchronize::is_at_safepoint(), "we can only safely fetch the displaced header at safepoint"); + if (m->has_displaced_mark_helper()) { + m = m->displaced_mark_helper(); + } + m = m->set_self_forwarded(); + assert(forwardee(m) == cast_to_oop(this), "encoding must be reversible"); + markOop old_mark = cas_set_mark_raw(m, compare, order); + if (old_mark == compare) { + return NULL; + } else { + assert(old_mark->is_marked(), "must be marked here"); + return forwardee(old_mark); + } +#else + return forward_to_atomic(cast_to_oop(this), order); +#endif +} + // Note that the forwardee is not the same thing as the displaced_mark. // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - return (oop) mark_raw()->decode_pointer(); + if (UseAltGCForwarding) { + return forwardee(mark()); + } else { + return (oop) mark_raw()->decode_pointer(); + } +} + +oop oopDesc::forwardee(markOop header) const { + assert(header->is_marked(), "only decode when actually forwarded"); +#ifdef _LP64 + if (header->self_forwarded()) { + return cast_to_oop(this); + } else +#endif + { + return cast_to_oop(header->decode_pointer()); + } } // Note that the forwardee is not the same thing as the displaced_mark. @@ -404,7 +444,11 @@ oop oopDesc::forwardee() const { // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee_acquire() const { markOop m = OrderAccess::load_acquire(&_mark); - return (oop) m->decode_pointer(); + if (UseAltGCForwarding) { + return forwardee(m); + } else { + return (oop) m->decode_pointer(); + } } // The following method needs to be MT safe. diff --git a/src/hotspot/share/runtime/arguments.cpp b/src/hotspot/share/runtime/arguments.cpp index d4cb3d0ee00..f08a73d1b01 100644 --- a/src/hotspot/share/runtime/arguments.cpp +++ b/src/hotspot/share/runtime/arguments.cpp @@ -4179,6 +4179,13 @@ jint Arguments::apply_ergo() { UseBiasedLocking = false; } + if (UseAltGCForwarding) { + if (!(UseG1GC || (UseParallelGC && UseParallelOldGC))) { + warning("UseAltGCForwarding is supported with current GC setting."); + FLAG_SET_DEFAULT(UseAltGCForwarding, false); + } + } + #ifdef CC_INTERP // Clear flags not supported on zero. FLAG_SET_DEFAULT(ProfileInterpreter, false); diff --git a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp index 77c2fe79d70..bcf4e4e119d 100644 --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp @@ -64,6 +64,8 @@ TEST_VM(PreservedMarks, iterate_and_restore) { FakeOop o3; FakeOop o4; + FlagSetting fs(UseAltGCForwarding, false); + // Make sure initial marks are correct. ASSERT_EQ(o1.mark(), FakeOop::originalMark()); ASSERT_EQ(o2.mark(), FakeOop::originalMark()); diff --git a/test/hotspot/gtest/gc/shared/test_slidingForwarding.cpp b/test/hotspot/gtest/gc/shared/test_slidingForwarding.cpp new file mode 100644 index 00000000000..06c80cbcf97 --- /dev/null +++ b/test/hotspot/gtest/gc/shared/test_slidingForwarding.cpp @@ -0,0 +1,127 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/shared/gc_globals.hpp" +#include "gc/shared/slidingForwarding.inline.hpp" +#include "oops/markOop.hpp" +#include "oops/oop.inline.hpp" +#include "utilities/align.hpp" +#include "unittest.hpp" + +#ifdef _LP64 +#ifndef PRODUCT + +static uintptr_t make_mark(uintptr_t target_region, uintptr_t offset) { + return (target_region) << 3 | (offset << 4) | 3 /* forwarded */; +} + +static uintptr_t make_fallback() { + return ((uintptr_t(1) << 2) /* fallback */ | 3 /* forwarded */); +} + +// Test simple forwarding within the same region. +TEST_VM(SlidingForwarding, simple) { + FlagSetting fs(UseAltGCForwarding, true); + + HeapWord* fakeheap[32] = { NULL }; + HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord)); + oop obj1 = cast_to_oop(&heap[2]); + oop obj2 = cast_to_oop(&heap[0]); + SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8); + obj1->set_mark(markOopDesc::prototype()); + SlidingForwarding::begin(); + + SlidingForwarding::forward_to(obj1, obj2); + ASSERT_EQ(obj1->mark()->value(), make_mark(0 /* target_region */, 0 /* offset */)); + ASSERT_EQ(SlidingForwarding::forwardee(obj1), obj2); + + SlidingForwarding::end(); +} + +// Test forwardings crossing 2 regions. +TEST_VM(SlidingForwarding, tworegions) { + FlagSetting fs(UseAltGCForwarding, true); + + HeapWord* fakeheap[32] = { NULL }; + HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord)); + oop obj1 = cast_to_oop(&heap[14]); + oop obj2 = cast_to_oop(&heap[2]); + oop obj3 = cast_to_oop(&heap[10]); + SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 8); + obj1->set_mark(markOopDesc::prototype()); + SlidingForwarding::begin(); + + SlidingForwarding::forward_to(obj1, obj2); + ASSERT_EQ(obj1->mark()->value(), make_mark(0 /* target_region */, 2 /* offset */)); + ASSERT_EQ(SlidingForwarding::forwardee(obj1), obj2); + + SlidingForwarding::forward_to(obj1, obj3); + ASSERT_EQ(obj1->mark()->value(), make_mark(1 /* target_region */, 2 /* offset */)); + ASSERT_EQ(SlidingForwarding::forwardee(obj1), obj3); + + SlidingForwarding::end(); +} + +// Test fallback forwardings crossing 4 regions. +TEST_VM(SlidingForwarding, fallback) { + FlagSetting fs(UseAltGCForwarding, true); + + HeapWord* fakeheap[32] = { NULL }; + HeapWord* heap = (HeapWord*)align_up(fakeheap, 8 * sizeof(HeapWord)); + oop s_obj1 = cast_to_oop(&heap[12]); + oop s_obj2 = cast_to_oop(&heap[13]); + oop s_obj3 = cast_to_oop(&heap[14]); + oop s_obj4 = cast_to_oop(&heap[15]); + oop t_obj1 = cast_to_oop(&heap[2]); + oop t_obj2 = cast_to_oop(&heap[4]); + oop t_obj3 = cast_to_oop(&heap[10]); + oop t_obj4 = cast_to_oop(&heap[12]); + SlidingForwarding::initialize(MemRegion(&heap[0], &heap[16]), 4); + s_obj1->set_mark(markOopDesc::prototype()); + s_obj2->set_mark(markOopDesc::prototype()); + s_obj3->set_mark(markOopDesc::prototype()); + s_obj4->set_mark(markOopDesc::prototype()); + SlidingForwarding::begin(); + + SlidingForwarding::forward_to(s_obj1, t_obj1); + ASSERT_EQ(s_obj1->mark()->value(), make_mark(0 /* target_region */, 2 /* offset */)); + ASSERT_EQ(SlidingForwarding::forwardee(s_obj1), t_obj1); + + SlidingForwarding::forward_to(s_obj2, t_obj2); + ASSERT_EQ(s_obj2->mark()->value(), make_mark(1 /* target_region */, 0 /* offset */)); + ASSERT_EQ(SlidingForwarding::forwardee(s_obj2), t_obj2); + + SlidingForwarding::forward_to(s_obj3, t_obj3); + ASSERT_EQ(s_obj3->mark()->value(), make_fallback()); + ASSERT_EQ(SlidingForwarding::forwardee(s_obj3), t_obj3); + + SlidingForwarding::forward_to(s_obj4, t_obj4); + ASSERT_EQ(s_obj4->mark()->value(), make_fallback()); + ASSERT_EQ(SlidingForwarding::forwardee(s_obj4), t_obj4); + + SlidingForwarding::end(); +} + +#endif // PRODUCT +#endif // _LP64