Split of "[SH] Add write barrier variants that support large allocation"

Differential Revision: D65701671

include/hermes/VM/AlignedHeapSegment.h

@@ -272,6 +272,17 @@ class AlignedHeapSegment {
     return (cp - base) >> LogHeapAlign;
   }
 
+  /// Return true if objects \p a and \p b live in the same segment. This is
+  /// used to check if a pointer field in an object points to another object in
+  /// the same segment (so that we don't need to dirty the cards -- we only need
+  /// to dirty cards that might contain old-to-young pointers, which must cross
+  /// segments). This also works for large segments, since there is only one
+  /// cell in those segments (i.e., \p a and \p b would be the same).
+  static bool containedInSameSegment(const GCCell *a, const GCCell *b) {
+    return (reinterpret_cast<uintptr_t>(a) ^ reinterpret_cast<uintptr_t>(b)) <
+        kSegmentUnitSize;
+  }
+
   /// Returns the index of the segment containing \p lowLim, which is required
   /// to be the start of its containing segment.  (This can allow extra
   /// efficiency, in cases where the segment start has already been computed.)

include/hermes/VM/GCBase.h

@@ -204,6 +204,14 @@ enum XorPtrKeyID {
 ///         const GCSmallHermesValue *start,
 ///         uint32_t numHVs);
 ///
+///   The above barriers may have a variant with "ForLargeObj" suffix, which is
+///   used when the heap location may be from a GCCell of a kind that supports
+///   large allocation. This variant is less efficient since it has to load the
+///   cards array through pointer in SHSegmentInfo, instead of the inline array
+///   field in CardTable structure. Because of this, these variant write
+///   barriers need a pointer to the start of the object in order to locate the
+///   card table for an object that is larger than the unit segment size.
+///
 ///   In debug builds: is a write barrier necessary for a write of the given
 ///   GC pointer \p value to the given \p loc?
 ///      bool needsWriteBarrier(void *loc, void *value);
@@ -1154,12 +1162,36 @@ class GCBase {
   /// Default implementations for read and write barriers: do nothing.
   void writeBarrier(const GCHermesValue *loc, HermesValue value);
   void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
   void writeBarrier(const GCPointerBase *loc, const GCCell *value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
   void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value);
   void constructorWriteBarrier(
       const GCSmallHermesValue *loc,
       SmallHermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
   void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
   void writeBarrierRange(const GCHermesValue *start, uint32_t numHVs);
   void writeBarrierRange(const GCSmallHermesValue *start, uint32_t numHVs);
   void constructorWriteBarrierRange(

include/hermes/VM/HadesGC.h

@@ -19,6 +19,7 @@
 #include "llvh/ADT/SparseBitVector.h"
 #include "llvh/Support/ErrorOr.h"
 #include "llvh/Support/PointerLikeTypeTraits.h"
+#include "llvh/Support/SaveAndRestore.h"
 
 #include <atomic>
 #include <condition_variable>
@@ -156,21 +157,67 @@ class HadesGC final : public GCBase {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCHermesValue *loc, HermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
 
   void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value) {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCSmallHermesValue *loc, SmallHermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
 
   /// The given pointer value is being written at the given loc (required to
   /// be in the heap). The value may be null. Execute a write barrier.
@@ -180,37 +227,134 @@ class HadesGC final : public GCBase {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCPointerBase *loc, const GCCell *value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
 
   /// Special versions of \p writeBarrier for when there was no previous value
   /// initialized into the space.
   void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       constructorWriteBarrierSlow(loc, value);
   }
   void constructorWriteBarrierSlow(const GCHermesValue *loc, HermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      constructorWriteBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void constructorWriteBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
 
   void constructorWriteBarrier(
       const GCSmallHermesValue *loc,
       SmallHermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       constructorWriteBarrierSlow(loc, value);
   }
   void constructorWriteBarrierSlow(
       const GCSmallHermesValue *loc,
       SmallHermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      constructorWriteBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void constructorWriteBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
 
   void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       relocationWriteBarrier(loc, value);
   }
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      relocationWriteBarrierForLargeObj(owningObj, loc, value);
+  }
 
   void constructorWriteBarrierRange(
       const GCCell *owningObj,
@@ -692,6 +836,14 @@ class HadesGC final : public GCBase {
   /// GC. This includes mark bits, free lists, etc.
   Mutex gcMutex_;
 
+#ifdef HERMES_SLOW_DEBUG
+  /// CellKind and size of the GCCell being constructed. It is set in makeA()
+  /// before constructing the cell and reset in the end. Note that we don't
+  /// use KindAndSize here since the actual size could be larger than
+  /// GCCell::maxSize().
+  llvh::Optional<std::pair<CellKind, uint32_t>> currentCellKindAndSize_;
+#endif
+
   /// Flag used to signal to the background thread that it should stop and yield
   /// the gcMutex_ to the mutator as soon as possible.
   AtomicIfConcurrentGC<bool> ogPaused_{false};
@@ -995,6 +1147,10 @@ class HadesGC final : public GCBase {
   /// pointers into YG and for tracking newly created pointers into the
   /// compactee.
   void relocationWriteBarrier(const void *loc, const void *value);
+  void relocationWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const void *loc,
+      const GCCell *value);
 
   /// Finalize all objects in YG that have finalizers.
   void finalizeYoungGenObjects();
@@ -1061,6 +1217,25 @@ class HadesGC final : public GCBase {
   void removeSegmentExtentFromCrashManager(const std::string &extraName);
 
 #ifdef HERMES_SLOW_DEBUG
+  /// Assert that \p loc does not belong to an object that supports large
+  /// allocation, for which we should call the other variant of write barrer.
+  void assertWriteBarrierForNonLargeObj(const void *loc);
+
+  /// Assert that \p loc does not belong to an object that supports large
+  /// allocation, for which we should call the other variant of constructor
+  /// write barrer.
+  void assertCtorWriteBarrierForNonLargeObj(const void *loc);
+
+  /// Assert that \p owningObj is an object that supports large allocation and
+  /// it contains \p loc.
+  void assertCtorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const void *loc);
+
+  /// Assert that \p owningObj is an object that supports large allocation and
+  /// it contains \p loc.
+  void assertWriteBarrierForLargeObj(const GCCell *owningObj, const void *loc);
+
   /// Checks the heap to make sure all cells are valid.
   void checkWellFormed();
 
@@ -1084,6 +1259,11 @@ inline T *HadesGC::makeA(uint32_t size, Args &&...args) {
       isSizeHeapAligned(size) &&
       "Call to makeA must use a size aligned to HeapAlign");
   assert(noAllocLevel_ == 0 && "No allocs allowed right now.");
+#ifdef HERMES_SLOW_DEBUG
+  llvh::SaveAndRestore<llvh::Optional<std::pair<CellKind, uint32_t>>>
+      saveCurrentCellKindAndSize(
+          currentCellKindAndSize_, std::make_pair(T::getCellKind(), size));
+#endif
   if (longLived == LongLived::Yes) {
     auto lk = ensureBackgroundTaskPaused();
     return constructCell<T>(

include/hermes/VM/MallocGC.h

@@ -235,10 +235,32 @@ class MallocGC final : public GCBase {
 
   void writeBarrier(const GCHermesValue *, HermesValue) {}
   void writeBarrier(const GCSmallHermesValue *, SmallHermesValue) {}
+  void
+  writeBarrierForLargeObj(const GCCell *, const GCHermesValue *, HermesValue) {}
+  void writeBarrierForLargeObj(
+      const GCCell *,
+      const GCSmallHermesValue *,
+      SmallHermesValue) {}
   void writeBarrier(const GCPointerBase *, const GCCell *) {}
+  void writeBarrierForLargeObj(
+      const GCCell *,
+      const GCPointerBase *,
+      const GCCell *) {}
   void constructorWriteBarrier(const GCHermesValue *, HermesValue) {}
   void constructorWriteBarrier(const GCSmallHermesValue *, SmallHermesValue) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCSmallHermesValue *,
+      SmallHermesValue) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCHermesValue *,
+      HermesValue) {}
   void constructorWriteBarrier(const GCPointerBase *, const GCCell *) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCPointerBase *,
+      const GCCell *) {}
   void writeBarrierRange(const GCHermesValue *, uint32_t) {}
   void writeBarrierRange(const GCSmallHermesValue *, uint32_t) {}
   void constructorWriteBarrierRange(