diff --git a/include/hermes/VM/AlignedHeapSegment.h b/include/hermes/VM/AlignedHeapSegment.h index d76912c9940..b5f09a02a6b 100644 --- a/include/hermes/VM/AlignedHeapSegment.h +++ b/include/hermes/VM/AlignedHeapSegment.h @@ -259,6 +259,16 @@ class AlignedHeapSegment { return (cp - base) >> LogHeapAlign; } + /// Return true if object \p a and \p b live in the same segment. This is used + /// to check if a pointer field in \p a may points to an object in the same + /// segment (so that we don't need to dirty the cards). This also works for + /// large segment, since there is only one cell in those segments (i.e., \p a + /// and \p b would be the same). + static bool containedInSame(const GCCell *a, const GCCell *b) { + return (reinterpret_cast(a) ^ reinterpret_cast(b)) < + kSegmentUnitSize; + } + /// Returns the index of the segment containing \p lowLim, which is required /// to be the start of its containing segment. (This can allow extra /// efficiency, in cases where the segment start has already been computed.) diff --git a/include/hermes/VM/GCBase.h b/include/hermes/VM/GCBase.h index 0692064e221..240c4553938 100644 --- a/include/hermes/VM/GCBase.h +++ b/include/hermes/VM/GCBase.h @@ -204,6 +204,14 @@ enum XorPtrKeyID { /// const GCSmallHermesValue *start, /// uint32_t numHVs); /// +/// The above barriers may have a variant with "ForLargeObj" suffix, which is +/// used when the heap location may be from a GCCell that supports large +/// allocation. This variant is less efficient since it has to load the cards +/// array through pointer in SHSegmentInfo, instead of the inline array field +/// in CardTable structure. This is necessary because the write barrier needs +/// a pointer to the start of the object in order to locate the card table for +/// an object that is larger than the segment alignment. +/// /// In debug builds: is a write barrier necessary for a write of the given /// GC pointer \p value to the given \p loc? /// bool needsWriteBarrier(void *loc, void *value); @@ -1154,12 +1162,36 @@ class GCBase { /// Default implementations for read and write barriers: do nothing. void writeBarrier(const GCHermesValue *loc, HermesValue value); void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void writeBarrier(const GCPointerBase *loc, const GCCell *value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value); void constructorWriteBarrier( const GCSmallHermesValue *loc, SmallHermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); void writeBarrierRange(const GCHermesValue *start, uint32_t numHVs); void writeBarrierRange(const GCSmallHermesValue *start, uint32_t numHVs); void constructorWriteBarrierRange( diff --git a/include/hermes/VM/HadesGC.h b/include/hermes/VM/HadesGC.h index e6686fd2e9d..eb0c4850700 100644 --- a/include/hermes/VM/HadesGC.h +++ b/include/hermes/VM/HadesGC.h @@ -156,21 +156,67 @@ class HadesGC final : public GCBase { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCHermesValue *loc, HermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); void writeBarrier(const GCSmallHermesValue *loc, SmallHermesValue value) { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCSmallHermesValue *loc, SmallHermesValue value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); /// The given pointer value is being written at the given loc (required to /// be in the heap). The value may be null. Execute a write barrier. @@ -180,24 +226,67 @@ class HadesGC final : public GCBase { assert( !calledByBackgroundThread() && "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) writeBarrierSlow(loc, value); } void writeBarrierSlow(const GCPointerBase *loc, const GCCell *value); + void writeBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + assert( + !calledByBackgroundThread() && + "Write barrier invoked by background thread."); +#ifdef HERMES_SLOW_DEBUG + assertWriteBarrierForLargeObj(owningObj, loc); +#endif + + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + writeBarrierSlowForLargeObj(owningObj, loc, value); + } + void writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value); /// Special versions of \p writeBarrier for when there was no previous value /// initialized into the space. void constructorWriteBarrier(const GCHermesValue *loc, HermesValue value) { +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) constructorWriteBarrierSlow(loc, value); } void constructorWriteBarrierSlow(const GCHermesValue *loc, HermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + constructorWriteBarrierSlowForLargeObj(owningObj, loc, value); + } + void constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value); void constructorWriteBarrier( const GCSmallHermesValue *loc, SmallHermesValue value) { +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) constructorWriteBarrierSlow(loc, value); @@ -205,12 +294,36 @@ class HadesGC final : public GCBase { void constructorWriteBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value); + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + constructorWriteBarrierSlowForLargeObj(owningObj, loc, value); + } + void constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value); void constructorWriteBarrier(const GCPointerBase *loc, const GCCell *value) { +#ifdef HERMES_SLOW_DEBUG + assertCtorWriteBarrierForNonLargeObj(loc); +#endif + // A pointer that lives in YG never needs any write barriers. if (LLVM_UNLIKELY(!inYoungGen(loc))) relocationWriteBarrier(loc, value); } + void constructorWriteBarrierForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + // A pointer that lives in YG never needs any write barriers. + if (LLVM_UNLIKELY(!inYoungGen(loc))) + relocationWriteBarrierForLargeObj(owningObj, loc, value); + } void constructorWriteBarrierRange( const GCCell *owningObj, @@ -995,6 +1108,10 @@ class HadesGC final : public GCBase { /// pointers into YG and for tracking newly created pointers into the /// compactee. void relocationWriteBarrier(const void *loc, const void *value); + void relocationWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc, + const GCCell *value); /// Finalize all objects in YG that have finalizers. void finalizeYoungGenObjects(); @@ -1061,6 +1178,18 @@ class HadesGC final : public GCBase { void removeSegmentExtentFromCrashManager(const std::string &extraName); #ifdef HERMES_SLOW_DEBUG + /// Assert that \p loc does not belong to an object that supports large + /// allocation, for which we should call the other variant of write barrer. + void assertWriteBarrierForNonLargeObj(const void *loc); + /// Assert that \p loc lives in a FixedSizeHeapSegment, this is a conservative + /// check since we may not get the CellKind when the owning object is still + /// being constructed. + void assertCtorWriteBarrierForNonLargeObj(const void *loc); + + /// Assert that \p owningObj is an object that supports large allocation and + /// it contains \p loc. + void assertWriteBarrierForLargeObj(const GCCell *owningObj, const void *loc); + /// Checks the heap to make sure all cells are valid. void checkWellFormed(); diff --git a/include/hermes/VM/MallocGC.h b/include/hermes/VM/MallocGC.h index 8e5e1b88277..ea0f2d9970b 100644 --- a/include/hermes/VM/MallocGC.h +++ b/include/hermes/VM/MallocGC.h @@ -235,10 +235,32 @@ class MallocGC final : public GCBase { void writeBarrier(const GCHermesValue *, HermesValue) {} void writeBarrier(const GCSmallHermesValue *, SmallHermesValue) {} + void + writeBarrierForLargeObj(const GCCell *, const GCHermesValue *, HermesValue) {} + void writeBarrierForLargeObj( + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue) {} void writeBarrier(const GCPointerBase *, const GCCell *) {} + void writeBarrierForLargeObj( + const GCCell *, + const GCPointerBase *, + const GCCell *) {} void constructorWriteBarrier(const GCHermesValue *, HermesValue) {} void constructorWriteBarrier(const GCSmallHermesValue *, SmallHermesValue) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCHermesValue *, + HermesValue) {} void constructorWriteBarrier(const GCPointerBase *, const GCCell *) {} + void constructorWriteBarrierForLargeObj( + const GCCell *, + const GCPointerBase *, + const GCCell *) {} void writeBarrierRange(const GCHermesValue *, uint32_t) {} void writeBarrierRange(const GCSmallHermesValue *, uint32_t) {} void constructorWriteBarrierRange( diff --git a/lib/VM/GCBase.cpp b/lib/VM/GCBase.cpp index a7e2b37dcd3..57172b39b4e 100644 --- a/lib/VM/GCBase.cpp +++ b/lib/VM/GCBase.cpp @@ -971,17 +971,47 @@ bool GCBase::shouldSanitizeHandles() { } GCBASE_BARRIER_2(writeBarrier, const GCHermesValue *, HermesValue); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCHermesValue *, + HermesValue); GCBASE_BARRIER_2(writeBarrier, const GCSmallHermesValue *, SmallHermesValue); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue); GCBASE_BARRIER_2(writeBarrier, const GCPointerBase *, const GCCell *); +GCBASE_BARRIER_3( + writeBarrierForLargeObj, + const GCCell *, + const GCPointerBase *, + const GCCell *); GCBASE_BARRIER_2(constructorWriteBarrier, const GCHermesValue *, HermesValue); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCHermesValue *, + HermesValue); GCBASE_BARRIER_2( constructorWriteBarrier, const GCSmallHermesValue *, SmallHermesValue); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCSmallHermesValue *, + SmallHermesValue); GCBASE_BARRIER_2( constructorWriteBarrier, const GCPointerBase *, const GCCell *); +GCBASE_BARRIER_3( + constructorWriteBarrierForLargeObj, + const GCCell *, + const GCPointerBase *, + const GCCell *); GCBASE_BARRIER_2(writeBarrierRange, const GCHermesValue *, uint32_t); GCBASE_BARRIER_2(writeBarrierRange, const GCSmallHermesValue *, uint32_t); GCBASE_BARRIER_3( diff --git a/lib/VM/gcs/HadesGC.cpp b/lib/VM/gcs/HadesGC.cpp index 343b53cdb9f..49daa37d9f3 100644 --- a/lib/VM/gcs/HadesGC.cpp +++ b/lib/VM/gcs/HadesGC.cpp @@ -13,6 +13,7 @@ #include "hermes/VM/CheckHeapWellFormedAcceptor.h" #include "hermes/VM/FillerCell.h" #include "hermes/VM/GCBase-inline.h" +#include "hermes/VM/GCCell.h" #include "hermes/VM/GCPointer.h" #include "hermes/VM/HermesValue-inline.h" #include "hermes/VM/RootAndSlotAcceptorDefault.h" @@ -1937,6 +1938,20 @@ void HadesGC::writeBarrierSlow(const GCHermesValue *loc, HermesValue value) { relocationWriteBarrier(loc, value.getPointer()); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + if (ogMarkingBarriers_) { + snapshotWriteBarrierInternal(*loc); + } + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, static_cast(value.getPointer())); +} + void HadesGC::writeBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value) { @@ -1949,6 +1964,20 @@ void HadesGC::writeBarrierSlow( relocationWriteBarrier(loc, value.getPointer(getPointerBase())); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + if (ogMarkingBarriers_) { + snapshotWriteBarrierInternal(*loc); + } + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, value.getPointer(getPointerBase())); +} + void HadesGC::writeBarrierSlow(const GCPointerBase *loc, const GCCell *value) { if (*loc && ogMarkingBarriers_) snapshotWriteBarrierInternal(*loc); @@ -1957,6 +1986,17 @@ void HadesGC::writeBarrierSlow(const GCPointerBase *loc, const GCCell *value) { relocationWriteBarrier(loc, value); } +void HadesGC::writeBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCPointerBase *loc, + const GCCell *value) { + if (*loc && ogMarkingBarriers_) + snapshotWriteBarrierInternal(*loc); + // Always do the non-snapshot write barrier in order for YG to be able to + // scan cards. + relocationWriteBarrierForLargeObj(owningObj, loc, value); +} + void HadesGC::constructorWriteBarrierSlow( const GCHermesValue *loc, HermesValue value) { @@ -1968,6 +2008,19 @@ void HadesGC::constructorWriteBarrierSlow( relocationWriteBarrier(loc, value.getPointer()); } +void HadesGC::constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCHermesValue *loc, + HermesValue value) { + // A constructor never needs to execute a SATB write barrier, since its + // previous value was definitely not live. + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, static_cast(value.getPointer())); +} + void HadesGC::constructorWriteBarrierSlow( const GCSmallHermesValue *loc, SmallHermesValue value) { @@ -1979,6 +2032,19 @@ void HadesGC::constructorWriteBarrierSlow( relocationWriteBarrier(loc, value.getPointer(getPointerBase())); } +void HadesGC::constructorWriteBarrierSlowForLargeObj( + const GCCell *owningObj, + const GCSmallHermesValue *loc, + SmallHermesValue value) { + // A constructor never needs to execute a SATB write barrier, since its + // previous value was definitely not live. + if (!value.isPointer()) { + return; + } + relocationWriteBarrierForLargeObj( + owningObj, loc, value.getPointer(getPointerBase())); +} + void HadesGC::constructorWriteBarrierRangeSlow( const GCCell *owningObj, const GCHermesValue *start, @@ -2100,6 +2166,20 @@ void HadesGC::relocationWriteBarrier(const void *loc, const void *value) { } } +void HadesGC::relocationWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc, + const GCCell *value) { + assert(!inYoungGen(loc) && "Pre-condition from other callers"); + if (AlignedHeapSegment::containedInSame(owningObj, value)) { + return; + } + if (inYoungGen(value) || compactee_.contains(value)) { + AlignedHeapSegment::cardTableCovering(owningObj) + ->dirtyCardForAddressInLargeObj(loc); + } +} + void HadesGC::weakRefReadBarrier(HermesValue value) { assert( !calledByBackgroundThread() && @@ -3112,6 +3192,54 @@ void HadesGC::removeSegmentExtentFromCrashManager( #ifdef HERMES_SLOW_DEBUG +void HadesGC::assertWriteBarrierForNonLargeObj(const void *loc) { + // Don't run the check with concurrent GC. Because when the background thread + // is sweeping, it may merge freed cells and poison memory outside of + // FreeListCell, which may be accessed by findObjectContaining() when visiting + // next cell. + if constexpr (!kConcurrentGC) { + auto *obj = + FixedSizeHeapSegment::cardTableCovering(loc)->findObjectContaining(loc); + assert( + !VTable::getVTable(obj->getKind())->allowLargeAlloc && + "writeBarrier() only works for GCCells that do not support large allocation"); + } else { + // With concurrent GC, we do the same conservative check as constructor + // write barrier. + auto segmentSize = + FixedSizeHeapSegment::cardTableCovering(loc)->getSegmentSize(); + assert( + (segmentSize == FixedSizeHeapSegment::kSize) && + "constructorWriteBarrier() does not work for GCCells larger than the size of FixedSizeHeapSegment"); + } +} + +void HadesGC::assertCtorWriteBarrierForNonLargeObj(const void *loc) { + // If constructorWriteBarrier() is called when the owning object is still + // being constructed, its CellKind is not set yet, so we can't assert that + // the owning object does not support large allocation. Instead, we check + // the size of segment covering this object. GCCells that support large + // allocation but get allocated in a normal segment would still work here. + auto segmentSize = + FixedSizeHeapSegment::cardTableCovering(loc)->getSegmentSize(); + assert( + (segmentSize == FixedSizeHeapSegment::kSize) && + "constructorWriteBarrier() does not work for GCCells larger than the size of FixedSizeHeapSegment"); +} + +void HadesGC::assertWriteBarrierForLargeObj( + const GCCell *owningObj, + const void *loc) { + assert( + VTable::getVTable(owningObj->getKind())->allowLargeAlloc && + "writeBarrierForLargeObj() should only used on GCCells that support large allocation"); + assert( + owningObj <= loc && + loc < (reinterpret_cast(owningObj) + + owningObj->getAllocatedSize()) && + "The owning object must contain the given heap location"); +} + void HadesGC::checkWellFormed() { CheckHeapWellFormedAcceptor acceptor(*this); {