diff --git a/include/hermes/VM/ArrayStorage.h b/include/hermes/VM/ArrayStorage.h index 8efec26a125..adc796d0cba 100644 --- a/include/hermes/VM/ArrayStorage.h +++ b/include/hermes/VM/ArrayStorage.h @@ -185,7 +185,7 @@ class ArrayStorageBase final assert(sz < capacity()); // Use the constructor of GCHermesValue to use the correct write barrier // for uninitialized memory. - new (&data()[sz]) GCHVType(value, runtime.getHeap()); + new (&data()[sz]) GCHVType(value, runtime.getHeap(), this); size_.store(sz + 1, std::memory_order_release); } diff --git a/include/hermes/VM/Callable.h b/include/hermes/VM/Callable.h index 5f63d34ccbd..5eccce24c3a 100644 --- a/include/hermes/VM/Callable.h +++ b/include/hermes/VM/Callable.h @@ -99,7 +99,8 @@ class Environment final getSlots(), getSlots() + size, HermesValue::encodeUndefinedValue(), - runtime.getHeap()); + runtime.getHeap(), + this); } /// Create an environment using the given function to retrieve the parent @@ -369,7 +370,8 @@ Environment::Environment( getSlots(), getSlots() + size, HermesValue::encodeUndefinedValue(), - runtime.getHeap()); + runtime.getHeap(), + this); } /// A function produced by Function.prototype.bind(). It packages a function diff --git a/include/hermes/VM/HermesValue-inline.h b/include/hermes/VM/HermesValue-inline.h index 0616c6a96ba..c247f9dcd3d 100644 --- a/include/hermes/VM/HermesValue-inline.h +++ b/include/hermes/VM/HermesValue-inline.h @@ -31,8 +31,13 @@ inline PinnedHermesValue &PinnedHermesValue::operator=(PseudoHandle &&hv) { template template -GCHermesValueBase::GCHermesValueBase(HVType hv, GC &gc) : HVType{hv} { +GCHermesValueBase::GCHermesValueBase( + HVType hv, + GC &gc, + const GCCell *cell) + : HVType{hv} { assert(!hv.isPointer() || hv.getPointer()); + (void)cell; if (NeedsBarriers::value) gc.constructorWriteBarrier(this, hv); } @@ -103,11 +108,12 @@ inline void GCHermesValueBase::uninitialized_fill( InputIt start, InputIt end, HVType fill, - GC &gc) { + GC &gc, + const GCCell *cell) { if (fill.isPointer()) { for (auto cur = start; cur != end; ++cur) { // Use the constructor write barrier. Assume it needs barriers. - new (&*cur) GCHermesValueBase(fill, gc); + new (&*cur) GCHermesValueBase(fill, gc, cell); } } else { for (auto cur = start; cur != end; ++cur) { @@ -141,13 +147,14 @@ inline OutputIt GCHermesValueBase::uninitialized_copy( InputIt first, InputIt last, OutputIt result, - GC &gc) { + GC &gc, + const GCCell *cell) { static_assert( !std::is_same::value || !std::is_same::value, "Pointer arguments must invoke pointer overload."); for (; first != last; ++first, (void)++result) { - new (&*result) GCHermesValueBase(*first, gc); + new (&*result) GCHermesValueBase(*first, gc, cell); } return result; } diff --git a/include/hermes/VM/HermesValue.h b/include/hermes/VM/HermesValue.h index 0dc3d12ad9d..1314c62fc57 100644 --- a/include/hermes/VM/HermesValue.h +++ b/include/hermes/VM/HermesValue.h @@ -523,7 +523,7 @@ class GCHermesValueBase final : public HVType { GCHermesValueBase() : HVType(HVType::encodeUndefinedValue()) {} /// Initialize a GCHermesValue from another HV. Performs a write barrier. template - GCHermesValueBase(HVType hv, GC &gc); + GCHermesValueBase(HVType hv, GC &gc, const GCCell *cell); /// Initialize a GCHermesValue from a non-pointer HV. Might perform a write /// barrier, depending on the GC. /// NOTE: The last parameter is unused, but acts as an overload selector. @@ -564,8 +564,12 @@ class GCHermesValueBase final : public HVType { /// been previously initialized. Cannot use this on previously initialized /// memory, as it will use an incorrect write barrier. template - static inline void - uninitialized_fill(InputIt first, InputIt last, HVType fill, GC &gc); + static inline void uninitialized_fill( + InputIt first, + InputIt last, + HVType fill, + GC &gc, + const GCCell *cell); /// Copies a range of values and performs a write barrier on each. template @@ -576,8 +580,12 @@ class GCHermesValueBase final : public HVType { /// been previously initialized. Cannot use this on previously initialized /// memory, as it will use an incorrect write barrier. template - static inline OutputIt - uninitialized_copy(InputIt first, InputIt last, OutputIt result, GC &gc); + static inline OutputIt uninitialized_copy( + InputIt first, + InputIt last, + OutputIt result, + GC &gc, + const GCCell *cell); /// Same as \p copy, but specialized for raw pointers. static inline GCHermesValueBase *copy( diff --git a/include/hermes/VM/JSObject.h b/include/hermes/VM/JSObject.h index baeb14732c0..441858d1be0 100644 --- a/include/hermes/VM/JSObject.h +++ b/include/hermes/VM/JSObject.h @@ -1676,7 +1676,8 @@ inline T *JSObject::initDirectPropStorage(Runtime &runtime, T *self) { self->directProps() + numOverlapSlots(), self->directProps() + DIRECT_PROPERTY_SLOTS, SmallHermesValue::encodeUndefinedValue(), - runtime.getHeap()); + runtime.getHeap(), + self); return self; } diff --git a/lib/VM/ArrayStorage.cpp b/lib/VM/ArrayStorage.cpp index 868a52448e8..9e1765ce3fe 100644 --- a/lib/VM/ArrayStorage.cpp +++ b/lib/VM/ArrayStorage.cpp @@ -112,7 +112,8 @@ ExecutionStatus ArrayStorageBase::reallocateToLarger( newSelf->data(), newSelf->data() + toFirst, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + newSelf); // Initialize the elements after the last copied element and toLast. if (toFirst + copySize < toLast) { @@ -120,7 +121,8 @@ ExecutionStatus ArrayStorageBase::reallocateToLarger( newSelf->data() + toFirst + copySize, newSelf->data() + toLast, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + newSelf); } newSelf->size_.store(toLast, std::memory_order_release); @@ -152,7 +154,8 @@ void ArrayStorageBase::resizeWithinCapacity( self->data() + sz, self->data() + newSize, HVType::encodeEmptyValue(), - gc); + gc, + self); } else if (newSize < sz) { // Execute write barriers on elements about to be conceptually changed to // null. @@ -212,7 +215,8 @@ ExecutionStatus ArrayStorageBase::shift( self->data() + toFirst + copySize, self->data() + toLast, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + self); } if (toLast < self->size()) { // Some elements are becoming unreachable, let the GC know. diff --git a/lib/VM/SegmentedArray.cpp b/lib/VM/SegmentedArray.cpp index 1a3a9cb4440..c46c7576106 100644 --- a/lib/VM/SegmentedArray.cpp +++ b/lib/VM/SegmentedArray.cpp @@ -62,7 +62,8 @@ void SegmentedArrayBase::Segment::setLength( data_ + len, data_ + newLength, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + this); length_.store(newLength, std::memory_order_release); } else if (newLength < len) { // If length is decreasing a write barrier needs to be done. @@ -193,8 +194,16 @@ ExecutionStatus SegmentedArrayBase::push_back( return ExecutionStatus::EXCEPTION; } const auto shv = HVType::encodeHermesValue(*value, runtime); - auto &elm = self->atRef(runtime, oldSize); - new (&elm) GCHVType(shv, runtime.getHeap()); + if (oldSize < kValueToSegmentThreshold) { + auto &elm = self->inlineStorage()[oldSize]; + new (&elm) GCHVType(shv, runtime.getHeap(), *self); + } else { + auto segmentNumber = toSegment(oldSize); + auto *segment = self->segmentAt(runtime, segmentNumber); + auto &elm = segment->at(toInterior(oldSize)); + // elm lives in segment, which is not the same cell as SegmentedArrayBase. + new (&elm) GCHVType(shv, runtime.getHeap(), segment); + } return ExecutionStatus::RETURNED; } @@ -395,7 +404,8 @@ void SegmentedArrayBase::increaseSizeWithinCapacity( inlineStorage() + currSize, inlineStorage() + finalSize, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + this); // Set the final size. numSlotsUsed_.store(finalSize, std::memory_order_release); return; @@ -410,7 +420,8 @@ void SegmentedArrayBase::increaseSizeWithinCapacity( inlineStorage() + currSize, inlineStorage() + kValueToSegmentThreshold, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + this); } segmentAt(runtime, segment)->setLength(runtime, segmentLength); } @@ -443,7 +454,8 @@ SegmentedArrayBase::increaseSize( self->inlineStorage() + currSize, self->inlineStorage() + kValueToSegmentThreshold, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + self.get()); // Set the size to the inline storage threshold. self->numSlotsUsed_.store( kValueToSegmentThreshold, std::memory_order_release); @@ -469,7 +481,8 @@ SegmentedArrayBase::increaseSize( self->numSlotsUsed_.load(std::memory_order_relaxed), self->inlineStorage() + newNumSlotsUsed, HVType::encodeEmptyValue(), - runtime.getHeap()); + runtime.getHeap(), + self.get()); self->numSlotsUsed_.store(newNumSlotsUsed, std::memory_order_release); // Allocate a handle to track the current array.