Split of "[SH] Add write barrier variants that support large allocation"

Differential Revision: D65701671

include/hermes/VM/AlignedHeapSegment.h

@@ -258,6 +258,16 @@ class AlignedHeapSegment {
     return (cp - base) >> LogHeapAlign;
   }
 
+  /// Return true if object \p a and \p b live in the same segment. This is used
+  /// to check if a pointer field in \p a may points to an object in the same
+  /// segment (so that we don't need to dirty the cards). This also works for
+  /// large segment, since there is only one cell in those segments (i.e., \p a
+  /// and \p b would be the same).
+  static bool containedInSame(const GCCell *a, const GCCell *b) {
+    return (reinterpret_cast<uintptr_t>(a) ^ reinterpret_cast<uintptr_t>(b)) <
+        kSegmentUnitSize;
+  }
+
   /// Returns the index of the segment containing \p lowLim, which is required
   /// to be the start of its containing segment.  (This can allow extra
   /// efficiency, in cases where the segment start has already been computed.)

include/hermes/VM/GCBase.h

@@ -204,6 +204,14 @@ enum XorPtrKeyID {
 ///         const GCSmallHermesValue *start,
 ///         uint32_t numHVs);
 ///
+///   The above barriers may have a variant with "ForLargeObj" suffix, which is
+///   used when the heap location may be from a GCCell that supports large
+///   allocation. This variant is less efficient since it has to load the cards
+///   array through pointer in SHSegmentInfo, instead of the inline array field
+///   in CardTable structure. This is necessary because the write barrier needs
+///   a pointer to the start of the object in order to locate the card table for
+///   an object that is larger than the segment alignment.
+///
 ///   In debug builds: is a write barrier necessary for a write of the given
 ///   GC pointer \p value to the given \p loc?
 ///      bool needsWriteBarrier(void *loc, void *value);
@@ -1154,12 +1162,36 @@ class GCBase {
   /// Default implementations for read and write barriers: do nothing.
   void writeBarrier(const GCHermesValue *loc, HermesValue value);
   void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
   void writeBarrier(const GCPointerBase *loc, const GCCell *value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
   void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value);
   void constructorWriteBarrier(
       const GCSmallHermesValue *loc,
       SmallHermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
   void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
   void writeBarrierRange(const GCHermesValue *start, uint32_t numHVs);
   void writeBarrierRange(const GCSmallHermesValue *start, uint32_t numHVs);
   void constructorWriteBarrierRange(

include/hermes/VM/HadesGC.h

@@ -156,21 +156,67 @@ class HadesGC final : public GCBase {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCHermesValue *loc, HermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
 
   void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value) {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCSmallHermesValue *loc, SmallHermesValue value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
 
   /// The given pointer value is being written at the given loc (required to
   /// be in the heap). The value may be null. Execute a write barrier.
@@ -180,37 +226,104 @@ class HadesGC final : public GCBase {
     assert(
         !calledByBackgroundThread() &&
         "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       writeBarrierSlow(loc, value);
   }
   void writeBarrierSlow(const GCPointerBase *loc, const GCCell *value);
+  void writeBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value) {
+    assert(
+        !calledByBackgroundThread() &&
+        "Write barrier invoked by background thread.");
+#ifdef HERMES_SLOW_DEBUG
+    assertWriteBarrierForLargeObj(owningObj, loc);
+#endif
+
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      writeBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void writeBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value);
 
   /// Special versions of \p writeBarrier for when there was no previous value
   /// initialized into the space.
   void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value) {
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       constructorWriteBarrierSlow(loc, value);
   }
   void constructorWriteBarrierSlow(const GCHermesValue *loc, HermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value) {
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      constructorWriteBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void constructorWriteBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCHermesValue *loc,
+      HermesValue value);
 
   void constructorWriteBarrier(
       const GCSmallHermesValue *loc,
       SmallHermesValue value) {
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       constructorWriteBarrierSlow(loc, value);
   }
   void constructorWriteBarrierSlow(
       const GCSmallHermesValue *loc,
       SmallHermesValue value);
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value) {
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      constructorWriteBarrierSlowForLargeObj(owningObj, loc, value);
+  }
+  void constructorWriteBarrierSlowForLargeObj(
+      const GCCell *owningObj,
+      const GCSmallHermesValue *loc,
+      SmallHermesValue value);
 
   void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value) {
+#ifdef HERMES_SLOW_DEBUG
+    assertCtorWriteBarrierForNonLargeObj(loc);
+#endif
+
     // A pointer that lives in YG never needs any write barriers.
     if (LLVM_UNLIKELY(!inYoungGen(loc)))
       relocationWriteBarrier(loc, value);
   }
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const GCPointerBase *loc,
+      const GCCell *value) {
+    // A pointer that lives in YG never needs any write barriers.
+    if (LLVM_UNLIKELY(!inYoungGen(loc)))
+      relocationWriteBarrierForLargeObj(owningObj, loc, value);
+  }
 
   void constructorWriteBarrierRange(
       const GCCell *owningObj,
@@ -995,6 +1108,10 @@ class HadesGC final : public GCBase {
   /// pointers into YG and for tracking newly created pointers into the
   /// compactee.
   void relocationWriteBarrier(const void *loc, const void *value);
+  void relocationWriteBarrierForLargeObj(
+      const GCCell *owningObj,
+      const void *loc,
+      const GCCell *value);
 
   /// Finalize all objects in YG that have finalizers.
   void finalizeYoungGenObjects();
@@ -1061,6 +1178,18 @@ class HadesGC final : public GCBase {
   void removeSegmentExtentFromCrashManager(const std::string &extraName);
 
 #ifdef HERMES_SLOW_DEBUG
+  /// Assert that \p loc does not belong to an object that supports large
+  /// allocation, for which we should call the other variant of write barrer.
+  void assertWriteBarrierForNonLargeObj(const void *loc);
+  /// Assert that \p loc lives in a FixedSizeHeapSegment, this is a conservative
+  /// check since we may not get the CellKind when the owning object is still
+  /// being constructed.
+  void assertCtorWriteBarrierForNonLargeObj(const void *loc);
+
+  /// Assert that \p owningObj is an object that supports large allocation and
+  /// it contains \p loc.
+  void assertWriteBarrierForLargeObj(const GCCell *owningObj, const void *loc);
+
   /// Checks the heap to make sure all cells are valid.
   void checkWellFormed();
 

include/hermes/VM/MallocGC.h

@@ -235,10 +235,32 @@ class MallocGC final : public GCBase {
 
   void writeBarrier(const GCHermesValue *, HermesValue) {}
   void writeBarrier(const GCSmallHermesValue *, SmallHermesValue) {}
+  void
+  writeBarrierForLargeObj(const GCCell *, const GCHermesValue *, HermesValue) {}
+  void writeBarrierForLargeObj(
+      const GCCell *,
+      const GCSmallHermesValue *,
+      SmallHermesValue) {}
   void writeBarrier(const GCPointerBase *, const GCCell *) {}
+  void writeBarrierForLargeObj(
+      const GCCell *,
+      const GCPointerBase *,
+      const GCCell *) {}
   void constructorWriteBarrier(const GCHermesValue *, HermesValue) {}
   void constructorWriteBarrier(const GCSmallHermesValue *, SmallHermesValue) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCSmallHermesValue *,
+      SmallHermesValue) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCHermesValue *,
+      HermesValue) {}
   void constructorWriteBarrier(const GCPointerBase *, const GCCell *) {}
+  void constructorWriteBarrierForLargeObj(
+      const GCCell *,
+      const GCPointerBase *,
+      const GCCell *) {}
   void writeBarrierRange(const GCHermesValue *, uint32_t) {}
   void writeBarrierRange(const GCSmallHermesValue *, uint32_t) {}
   void constructorWriteBarrierRange(

lib/VM/GCBase.cpp

@@ -971,17 +971,47 @@ bool GCBase::shouldSanitizeHandles() {
   }
 
 GCBASE_BARRIER_2(writeBarrier, const GCHermesValue *, HermesValue);
+GCBASE_BARRIER_3(
+    writeBarrierForLargeObj,
+    const GCCell *,
+    const GCHermesValue *,
+    HermesValue);
 GCBASE_BARRIER_2(writeBarrier, const GCSmallHermesValue *, SmallHermesValue);
+GCBASE_BARRIER_3(
+    writeBarrierForLargeObj,
+    const GCCell *,
+    const GCSmallHermesValue *,
+    SmallHermesValue);
 GCBASE_BARRIER_2(writeBarrier, const GCPointerBase *, const GCCell *);
+GCBASE_BARRIER_3(
+    writeBarrierForLargeObj,
+    const GCCell *,
+    const GCPointerBase *,
+    const GCCell *);
 GCBASE_BARRIER_2(constructorWriteBarrier, const GCHermesValue *, HermesValue);
+GCBASE_BARRIER_3(
+    constructorWriteBarrierForLargeObj,
+    const GCCell *,
+    const GCHermesValue *,
+    HermesValue);
 GCBASE_BARRIER_2(
     constructorWriteBarrier,
     const GCSmallHermesValue *,
     SmallHermesValue);
+GCBASE_BARRIER_3(
+    constructorWriteBarrierForLargeObj,
+    const GCCell *,
+    const GCSmallHermesValue *,
+    SmallHermesValue);
 GCBASE_BARRIER_2(
     constructorWriteBarrier,
     const GCPointerBase *,
     const GCCell *);
+GCBASE_BARRIER_3(
+    constructorWriteBarrierForLargeObj,
+    const GCCell *,
+    const GCPointerBase *,
+    const GCCell *);
 GCBASE_BARRIER_2(writeBarrierRange, const GCHermesValue *, uint32_t);
 GCBASE_BARRIER_2(writeBarrierRange, const GCSmallHermesValue *, uint32_t);
 GCBASE_BARRIER_3(