[Backport] 8305896: Alternative full GC forwarding

Summary: Introduce UseAltGCForwarding to support alternative full GC forwarding.
	 Now support G1 and parallelGC. The code accords to 8305896 and https://github.com/openjdk/lilliput-jdk17u/

Test Plan: CICD

Reviewed-by: yude.lin, yifeng.jin

Issue: #683

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -71,6 +71,7 @@
 #include "gc/shared/isGCActiveMark.hpp"
 #include "gc/shared/oopStorageParState.hpp"
 #include "gc/shared/preservedMarks.inline.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/shared/referenceProcessor.inline.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
@@ -1734,6 +1735,10 @@ jint G1CollectedHeap::initialize() {
 
   _collection_set.initialize(max_regions());
 
+  if (UseAltGCForwarding) {
+    SlidingForwarding::initialize(MemRegion((HeapWord*)heap_rs.base(), (HeapWord*)heap_rs.end()), HeapRegion::GrainWords);
+  }
+
   return JNI_OK;
 }
 

src/hotspot/share/gc/g1/g1FullCollector.cpp

@@ -41,6 +41,7 @@
 #include "gc/shared/gcTraceTime.inline.hpp"
 #include "gc/shared/preservedMarks.hpp"
 #include "gc/shared/referenceProcessor.hpp"
+#include "gc/shared/slidingForwarding.hpp"
 #include "gc/shared/weakProcessor.hpp"
 #include "logging/log.hpp"
 #include "runtime/biasedLocking.hpp"
@@ -171,11 +172,19 @@ void G1FullCollector::collect() {
   // Don't add any more derived pointers during later phases
   deactivate_derived_pointers();
 
+  if (UseAltGCForwarding) {
+    SlidingForwarding::begin();
+  }
+
   phase2_prepare_compaction();
 
   phase3_adjust_pointers();
 
   phase4_do_compaction();
+
+  if (UseAltGCForwarding) {
+    SlidingForwarding::end();
+  }
 }
 
 void G1FullCollector::complete_collection() {
@@ -248,8 +257,15 @@ void G1FullCollector::phase3_adjust_pointers() {
   // Adjust the pointers to reflect the new locations
   GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer());
 
-  G1FullGCAdjustTask task(this);
-  run_task(&task);
+  if (UseAltGCForwarding) {
+    G1AdjustClosure<true> adjust;
+    G1FullGCAdjustTask task(this, &adjust);
+    run_task(&task);
+  } else {
+    G1AdjustClosure<false> adjust;
+    G1FullGCAdjustTask task(this, &adjust);
+    run_task(&task);
+  }
 }
 
 void G1FullCollector::phase4_do_compaction() {

src/hotspot/share/gc/g1/g1FullGCAdjustTask.cpp

@@ -36,17 +36,19 @@
 #include "logging/log.hpp"
 #include "memory/iterator.inline.hpp"
 
+template <bool ALT_FWD>
 class G1AdjustLiveClosure : public StackObj {
-  G1AdjustClosure* _adjust_closure;
+  G1AdjustClosure<ALT_FWD>* _adjust_closure;
 public:
-  G1AdjustLiveClosure(G1AdjustClosure* cl) :
+  G1AdjustLiveClosure(G1AdjustClosure<ALT_FWD>* cl) :
     _adjust_closure(cl) { }
 
   size_t apply(oop object) {
     return object->oop_iterate_size(_adjust_closure);
   }
 };
 
+template <bool ALT_FWD>
 class G1AdjustRegionClosure : public HeapRegionClosure {
   G1CMBitMap* _bitmap;
   uint _worker_id;
@@ -56,32 +58,32 @@ class G1AdjustRegionClosure : public HeapRegionClosure {
     _worker_id(worker_id) { }
 
   bool do_heap_region(HeapRegion* r) {
-    G1AdjustClosure cl;
+    G1AdjustClosure<ALT_FWD> cl;
     if (r->is_humongous()) {
       oop obj = oop(r->humongous_start_region()->bottom());
       obj->oop_iterate(&cl, MemRegion(r->bottom(), r->top()));
     } else if (r->is_open_archive()) {
       // Only adjust the open archive regions, the closed ones
       // never change.
-      G1AdjustLiveClosure adjust(&cl);
+      G1AdjustLiveClosure<ALT_FWD> adjust(&cl);
       r->apply_to_marked_objects(_bitmap, &adjust);
       // Open archive regions will not be compacted and the marking information is
       // no longer needed. Clear it here to avoid having to do it later.
       _bitmap->clear_region(r);
     } else {
-      G1AdjustLiveClosure adjust(&cl);
+      G1AdjustLiveClosure<ALT_FWD> adjust(&cl);
       r->apply_to_marked_objects(_bitmap, &adjust);
     }
     return false;
   }
 };
 
-G1FullGCAdjustTask::G1FullGCAdjustTask(G1FullCollector* collector) :
+G1FullGCAdjustTask::G1FullGCAdjustTask(G1FullCollector* collector, OopClosure* adjust) :
     G1FullGCTask("G1 Adjust", collector),
     _root_processor(G1CollectedHeap::heap(), collector->workers()),
     _hrclaimer(collector->workers()),
-    _adjust(),
-    _adjust_string_dedup(NULL, &_adjust, G1StringDedup::is_enabled()) {
+    _adjust(adjust),
+    _adjust_string_dedup(NULL, _adjust, G1StringDedup::is_enabled()) {
   // Need cleared claim bits for the roots processing
   ClassLoaderDataGraph::clear_claimed_marks();
 }
@@ -95,13 +97,13 @@ void G1FullGCAdjustTask::work(uint worker_id) {
   marker->preserved_stack()->adjust_during_full_gc();
 
   // Adjust the weak_roots.
-  CLDToOopClosure adjust_cld(&_adjust);
-  CodeBlobToOopClosure adjust_code(&_adjust, CodeBlobToOopClosure::FixRelocations);
-  _root_processor.process_full_gc_weak_roots(&_adjust);
+  CLDToOopClosure adjust_cld(_adjust);
+  CodeBlobToOopClosure adjust_code(_adjust, CodeBlobToOopClosure::FixRelocations);
+  _root_processor.process_full_gc_weak_roots(_adjust);
 
   // Needs to be last, process_all_roots calls all_tasks_completed(...).
   _root_processor.process_all_roots(
-      &_adjust,
+      _adjust,
       &adjust_cld,
       &adjust_code);
 
@@ -111,7 +113,12 @@ void G1FullGCAdjustTask::work(uint worker_id) {
   }
 
   // Now adjust pointers region by region
-  G1AdjustRegionClosure blk(collector()->mark_bitmap(), worker_id);
-  G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
+  if (UseAltGCForwarding) {
+    G1AdjustRegionClosure<true> blk(collector()->mark_bitmap(), worker_id);
+    G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
+  } else {
+    G1AdjustRegionClosure<false> blk(collector()->mark_bitmap(), worker_id);
+    G1CollectedHeap::heap()->heap_region_par_iterate_from_worker_offset(&blk, &_hrclaimer, worker_id);
+  }
   log_task("Adjust task", worker_id, start);
 }

src/hotspot/share/gc/g1/g1FullGCAdjustTask.hpp

@@ -37,11 +37,11 @@ class G1CollectedHeap;
 class G1FullGCAdjustTask : public G1FullGCTask {
   G1RootProcessor          _root_processor;
   HeapRegionClaimer        _hrclaimer;
-  G1AdjustClosure          _adjust;
+  OopClosure*              _adjust;
   G1StringDedupUnlinkOrOopsDoClosure _adjust_string_dedup;
 
 public:
-  G1FullGCAdjustTask(G1FullCollector* collector);
+  G1FullGCAdjustTask(G1FullCollector* collector, OopClosure* adjust);
   void work(uint worker_id);
 };
 

src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1FullGCCompactTask.hpp"
 #include "gc/g1/heapRegion.inline.hpp"
 #include "gc/shared/gcTraceTime.inline.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "logging/log.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/ticks.hpp"
@@ -59,12 +60,21 @@ class G1ResetHumongousClosure : public HeapRegionClosure {
   }
 };
 
-size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
+template <bool ALT_FWD>
+size_t G1FullGCCompactTask::G1CompactRegionClosure<ALT_FWD>::apply(oop obj) {
   size_t size = obj->size();
-  HeapWord* destination = (HeapWord*)obj->forwardee();
-  if (destination == NULL) {
-    // Object not moving
-    return size;
+  HeapWord* destination;
+  if (ALT_FWD) {
+    if (!SlidingForwarding::is_forwarded(obj)) {
+      return size;
+    }
+    destination = cast_from_oop<HeapWord*>(SlidingForwarding::forwardee<true>(obj));
+  } else {
+    destination = (HeapWord*)obj->forwardee();
+    if (destination == NULL) {
+      // Object not moving
+      return size;
+    }
   }
 
   // copy object and reinit its mark
@@ -79,8 +89,13 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) {
 
 void G1FullGCCompactTask::compact_region(HeapRegion* hr) {
   assert(!hr->is_humongous(), "Should be no humongous regions in compaction queue");
-  G1CompactRegionClosure compact(collector()->mark_bitmap());
-  hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+  if (UseAltGCForwarding) {
+    G1CompactRegionClosure<true> compact(collector()->mark_bitmap());
+    hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+  } else {
+    G1CompactRegionClosure<false> compact(collector()->mark_bitmap());
+    hr->apply_to_marked_objects(collector()->mark_bitmap(), &compact);
+  }
   // Once all objects have been moved the liveness information
   // needs be cleared.
   collector()->mark_bitmap()->clear_region(hr);

src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp

@@ -49,6 +49,7 @@ class G1FullGCCompactTask : public G1FullGCTask {
   void work(uint worker_id);
   void serial_compaction();
 
+  template <bool ALT_FWD>
   class G1CompactRegionClosure : public StackObj {
     G1CMBitMap* _bitmap;
 

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/g1/g1FullGCCompactionPoint.hpp"
 #include "gc/g1/heapRegion.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/debug.hpp"
 
@@ -93,6 +94,7 @@ void G1FullGCCompactionPoint::switch_region() {
   initialize_values(true);
 }
 
+template <bool ALT_FWD>
 void G1FullGCCompactionPoint::forward(oop object, size_t size) {
   assert(_current_region != NULL, "Must have been initialized");
 
@@ -103,26 +105,34 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
 
   // Store a forwarding pointer if the object should be moved.
   if ((HeapWord*)object != _compaction_top) {
-    object->forward_to(oop(_compaction_top));
+    if (ALT_FWD) {
+      SlidingForwarding::forward_to<true>(object, cast_to_oop(_compaction_top));
+    } else {
+      object->forward_to(oop(_compaction_top));
+    }
   } else {
-    if (object->forwardee() != NULL) {
-      // Object should not move but mark-word is used so it looks like the
-      // object is forwarded. Need to clear the mark and it's no problem
-      // since it will be restored by preserved marks. There is an exception
-      // with BiasedLocking, in this case forwardee() will return NULL
-      // even if the mark-word is used. This is no problem since
-      // forwardee() will return NULL in the compaction phase as well.
-      object->init_mark_raw();
+    if (ALT_FWD) {
+      assert(!SlidingForwarding::is_forwarded(object), "should not be forwarded");
     } else {
-      // Make sure object has the correct mark-word set or that it will be
-      // fixed when restoring the preserved marks.
-      assert(object->mark_raw() == markOopDesc::prototype_for_object(object) || // Correct mark
-             object->mark_raw()->must_be_preserved(object) || // Will be restored by PreservedMarksSet
-             (UseBiasedLocking && object->has_bias_pattern_raw()), // Will be restored by BiasedLocking
-             "should have correct prototype obj: " PTR_FORMAT " mark: " PTR_FORMAT " prototype: " PTR_FORMAT,
-             p2i(object), p2i(object->mark_raw()), p2i(markOopDesc::prototype_for_object(object)));
+      if (object->forwardee() != NULL) {
+        // Object should not move but mark-word is used so it looks like the
+        // object is forwarded. Need to clear the mark and it's no problem
+        // since it will be restored by preserved marks. There is an exception
+        // with BiasedLocking, in this case forwardee() will return NULL
+        // even if the mark-word is used. This is no problem since
+        // forwardee() will return NULL in the compaction phase as well.
+        object->init_mark_raw();
+      } else {
+        // Make sure object has the correct mark-word set or that it will be
+        // fixed when restoring the preserved marks.
+        assert(object->mark_raw() == markOopDesc::prototype_for_object(object) || // Correct mark
+               object->mark_raw()->must_be_preserved(object) || // Will be restored by PreservedMarksSet
+               (UseBiasedLocking && object->has_bias_pattern_raw()), // Will be restored by BiasedLocking
+               "should have correct prototype obj: " PTR_FORMAT " mark: " PTR_FORMAT " prototype: " PTR_FORMAT,
+               p2i(object), p2i(object->mark_raw()), p2i(markOopDesc::prototype_for_object(object)));
+      }
+      assert(object->forwardee() == NULL, "should be forwarded to NULL");
     }
-    assert(object->forwardee() == NULL, "should be forwarded to NULL");
   }
 
   // Update compaction values.
@@ -132,6 +142,9 @@ void G1FullGCCompactionPoint::forward(oop object, size_t size) {
   }
 }
 
+template void G1FullGCCompactionPoint::forward<true>(oop object, size_t size);
+template void G1FullGCCompactionPoint::forward<false>(oop object, size_t size);
+
 void G1FullGCCompactionPoint::add(HeapRegion* hr) {
   _compaction_regions->append(hr);
 }

src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp

@@ -51,6 +51,7 @@ class G1FullGCCompactionPoint : public CHeapObj<mtGC> {
   bool is_initialized();
   void initialize(HeapRegion* hr, bool init_threshold);
   void update();
+  template <bool ALT_FWD>
   void forward(oop object, size_t size);
   void add(HeapRegion* hr);
   void merge(G1FullGCCompactionPoint* other);

src/hotspot/share/gc/g1/g1FullGCOopClosures.hpp

@@ -74,6 +74,7 @@ class G1MarkAndPushClosure : public OopIterateClosure {
   virtual void do_cld(ClassLoaderData* cld);
 };
 
+template <bool ALT_FWD>
 class G1AdjustClosure : public BasicOopIterateClosure {
   template <class T> static inline void adjust_pointer(T* p);
 public:

src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1FullGCMarker.inline.hpp"
 #include "gc/g1/g1FullGCOopClosures.hpp"
 #include "gc/g1/heapRegionRemSet.hpp"
+#include "gc/shared/slidingForwarding.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -60,7 +61,8 @@ inline void G1MarkAndPushClosure::do_cld(ClassLoaderData* cld) {
   _marker->follow_cld(cld);
 }
 
-template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
+template <bool ALT_FWD>
+template <class T> inline void G1AdjustClosure<ALT_FWD>::adjust_pointer(T* p) {
   T heap_oop = RawAccess<>::oop_load(p);
   if (CompressedOops::is_null(heap_oop)) {
     return;
@@ -73,24 +75,33 @@ template <class T> inline void G1AdjustClosure::adjust_pointer(T* p) {
     return;
   }
 
-  oop forwardee = obj->forwardee();
-  if (forwardee == NULL) {
-    // Not forwarded, return current reference.
-    assert(obj->mark_raw() == markOopDesc::prototype_for_object(obj) || // Correct mark
-           obj->mark_raw()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet
-           (UseBiasedLocking && obj->has_bias_pattern_raw()), // Will be restored by BiasedLocking
-           "Must have correct prototype or be preserved, obj: " PTR_FORMAT ", mark: " PTR_FORMAT ", prototype: " PTR_FORMAT,
-           p2i(obj), p2i(obj->mark_raw()), p2i(markOopDesc::prototype_for_object(obj)));
-    return;
+  oop forwardee = NULL;
+  if (ALT_FWD) {
+    if (!SlidingForwarding::is_forwarded(obj)) {
+      return;
+    }
+    forwardee = SlidingForwarding::forwardee<true>(obj);
+  } else {
+    forwardee = obj->forwardee();
+    if (forwardee == NULL) {
+      assert(obj->mark_raw() == markOopDesc::prototype_for_object(obj) || // Correct mark
+             obj->mark_raw()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet
+             (UseBiasedLocking && obj->has_bias_pattern_raw()), // Will be restored by BiasedLocking
+             "Must have correct prototype or be preserved, obj: " PTR_FORMAT ", mark: " PTR_FORMAT ", prototype: " PTR_FORMAT,
+             p2i(obj), p2i(obj->mark_raw()), p2i(markOopDesc::prototype_for_object(obj)));
+      return;
+    }
   }
 
   // Forwarded, just update.
   assert(Universe::heap()->is_in_reserved(forwardee), "should be in object space");
   RawAccess<IS_NOT_NULL>::oop_store(p, forwardee);
 }
 
-inline void G1AdjustClosure::do_oop(oop* p)       { do_oop_work(p); }
-inline void G1AdjustClosure::do_oop(narrowOop* p) { do_oop_work(p); }
+template <bool ALT_FWD>
+inline void G1AdjustClosure<ALT_FWD>::do_oop(oop* p)       { do_oop_work(p); }
+template <bool ALT_FWD>
+inline void G1AdjustClosure<ALT_FWD>::do_oop(narrowOop* p) { do_oop_work(p); }
 
 inline bool G1IsAliveClosure::do_object_b(oop p) {
   return _bitmap->is_marked(p) || G1ArchiveAllocator::is_closed_archive_object(p);