Implement frozen object heap

src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs

@@ -153,7 +153,7 @@ private static unsafe void InitializeGlobalTablesForModule(TypeManagerHandle typ
 
         private static unsafe void InitializeModuleFrozenObjectSegment(IntPtr segmentStart, int length)
         {
-            if (RuntimeImports.RhpRegisterFrozenSegment(segmentStart, (IntPtr)length) == IntPtr.Zero)
+            if (RuntimeImports.RhRegisterFrozenSegment((void*)segmentStart, (nuint)length, (nuint)length, (nuint)length) == IntPtr.Zero)
             {
                 // This should only happen if we ran out of memory.
                 RuntimeExceptionHelpers.FailFast("Failed to register frozen object segment for the module.");

src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeImports.cs

@@ -13,14 +13,6 @@ internal static class RuntimeImports
     {
         private const string RuntimeLibrary = "*";
 
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        [RuntimeImport(RuntimeLibrary, "RhpRegisterFrozenSegment")]
-        internal static extern IntPtr RhpRegisterFrozenSegment(IntPtr pSegmentStart, IntPtr length);
-
-        [MethodImpl(MethodImplOptions.InternalCall)]
-        [RuntimeImport(RuntimeLibrary, "RhpUnregisterFrozenSegment")]
-        internal static extern void RhpUnregisterFrozenSegment(IntPtr pSegmentHandle);
-
         [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")]
         [MethodImplAttribute(MethodImplOptions.InternalCall)]
         private static extern IntPtr RhGetModuleSection(ref TypeManagerHandle module, ReadyToRunSectionType section, out int length);

src/coreclr/nativeaot/Runtime/MiscHelpers.cpp

@@ -356,12 +356,17 @@ EXTERN_C NATIVEAOT_API int32_t __cdecl RhpGetCurrentThreadStackTrace(void* pOutp
     return RhpCalculateStackTraceWorker(pOutputBuffer, outputBufferLength, pAddressInCurrentFrame);
 }
 
-COOP_PINVOKE_HELPER(void*, RhpRegisterFrozenSegment, (void* pSegmentStart, size_t length))
+EXTERN_C NATIVEAOT_API void* __cdecl RhRegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize)
 {
-    return RedhawkGCInterface::RegisterFrozenSegment(pSegmentStart, length);
+    return RedhawkGCInterface::RegisterFrozenSegment(pSection, allocSize, commitSize, reservedSize);
 }
 
-COOP_PINVOKE_HELPER(void, RhpUnregisterFrozenSegment, (void* pSegmentHandle))
+EXTERN_C NATIVEAOT_API void __cdecl RhUpdateFrozenSegment(void* pSegmentHandle, uint8_t* allocated, uint8_t* committed)
+{
+    RedhawkGCInterface::UpdateFrozenSegment((GcSegmentHandle)pSegmentHandle, allocated, committed);
+}
+
+EXTERN_C NATIVEAOT_API void __cdecl RhUnregisterFrozenSegment(void* pSegmentHandle)
 {
     RedhawkGCInterface::UnregisterFrozenSegment((GcSegmentHandle)pSegmentHandle);
 }

src/coreclr/nativeaot/Runtime/gcrhenv.cpp

@@ -368,23 +368,34 @@ void RedhawkGCInterface::BulkEnumGcObjRef(PTR_RtuObjectRef pRefs, uint32_t cRefs
 }
 
 // static
-GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_t SizeSection)
+GcSegmentHandle RedhawkGCInterface::RegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize)
 {
+    ASSERT(allocSize <= commitSize);
+    ASSERT(commitSize <= reservedSize);
+
 #ifdef FEATURE_BASICFREEZE
     segment_info seginfo;
 
     seginfo.pvMem           = pSection;
     seginfo.ibFirstObject   = sizeof(ObjHeader);
-    seginfo.ibAllocated     = SizeSection;
-    seginfo.ibCommit        = seginfo.ibAllocated;
-    seginfo.ibReserved      = seginfo.ibAllocated;
+    seginfo.ibAllocated     = allocSize;
+    seginfo.ibCommit        = commitSize;
+    seginfo.ibReserved      = reservedSize;
 
     return (GcSegmentHandle)GCHeapUtilities::GetGCHeap()->RegisterFrozenSegment(&seginfo);
 #else // FEATURE_BASICFREEZE
     return NULL;
 #endif // FEATURE_BASICFREEZE
 }
 
+// static
+void RedhawkGCInterface::UpdateFrozenSegment(GcSegmentHandle seg, uint8_t* allocated, uint8_t* committed)
+{
+    ASSERT(allocated <= committed);
+
+    GCHeapUtilities::GetGCHeap()->UpdateFrozenSegment((segment_handle)seg, allocated, committed);
+}
+
 // static
 void RedhawkGCInterface::UnregisterFrozenSegment(GcSegmentHandle segment)
 {

src/coreclr/nativeaot/Runtime/gcrhinterface.h

@@ -129,7 +129,8 @@ class RedhawkGCInterface
                                                  void * pfnEnumCallback,
                                                  void * pvCallbackData);
 
-    static GcSegmentHandle RegisterFrozenSegment(void * pSection, size_t SizeSection);
+    static GcSegmentHandle RegisterFrozenSegment(void * pSection, size_t allocSize, size_t commitSize, size_t reservedSize);
+    static void UpdateFrozenSegment(GcSegmentHandle seg, uint8_t* allocated, uint8_t* committed);
     static void UnregisterFrozenSegment(GcSegmentHandle segment);
 
 #ifdef FEATURE_GC_STRESS

src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Unix.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+
+namespace Internal.Runtime
+{
+    internal unsafe partial class FrozenObjectHeapManager
+    {
+        static void* ClrVirtualReserve(nuint size)
+        {
+            // The shim will return null for failure
+            return (void*)Interop.Sys.MMap(
+                0,
+                size,
+                Interop.Sys.MemoryMappedProtections.PROT_NONE,
+                Interop.Sys.MemoryMappedFlags.MAP_PRIVATE | Interop.Sys.MemoryMappedFlags.MAP_ANONYMOUS,
+                -1,
+                0);
+
+        }
+
+        static void* ClrVirtualCommit(void* pBase, nuint size)
+        {
+            int result = Interop.Sys.MProtect(
+                (nint)pBase,
+                size,
+                Interop.Sys.MemoryMappedProtections.PROT_READ | Interop.Sys.MemoryMappedProtections.PROT_WRITE);
+
+            return result == 0 ? pBase : null;
+        }
+
+        static void ClrVirtualFree(void* pBase, nuint size)
+        {
+            Debug.Assert(size != 0);
+            Interop.Sys.MUnmap((nint)pBase, size);
+        }
+    }
+}

src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.Windows.cs

@@ -0,0 +1,31 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+
+namespace Internal.Runtime
+{
+    internal unsafe partial class FrozenObjectHeapManager
+    {
+        static void* ClrVirtualReserve(nuint size)
+        {
+            return Interop.Kernel32.VirtualAlloc(null, size, Interop.Kernel32.MemOptions.MEM_RESERVE, Interop.Kernel32.PageOptions.PAGE_READWRITE);
+        }
+
+        static void* ClrVirtualCommit(void* pBase, nuint size)
+        {
+            return Interop.Kernel32.VirtualAlloc(pBase, size, Interop.Kernel32.MemOptions.MEM_COMMIT, Interop.Kernel32.PageOptions.PAGE_READWRITE);
+        }
+
+        static void ClrVirtualFree(void* pBase, nuint size)
+        {
+            // We require the size parameter for Unix implementation sake.
+            // The Win32 API ignores this parameter because we must pass zero.
+            // If the caller passed zero, this is going to be broken on Unix
+            // so let's at least assert that.
+            Debug.Assert(size != 0);
+
+            Interop.Kernel32.VirtualFree(pBase, 0, Interop.Kernel32.MemOptions.MEM_RELEASE);
+        }
+    }
+}

src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/FrozenObjectHeapManager.cs

@@ -0,0 +1,211 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Runtime;
+using System.Runtime.CompilerServices;
+using System.Threading;
+
+using Debug = System.Diagnostics.Debug;
+
+namespace Internal.Runtime
+{
+    internal unsafe partial class FrozenObjectHeapManager
+    {
+        public static readonly FrozenObjectHeapManager Instance = new FrozenObjectHeapManager();
+
+        private readonly LowLevelLock m_Crst = new LowLevelLock();
+        private FrozenObjectSegment m_CurrentSegment;
+
+        // Default size to reserve for a frozen segment
+        private const nuint FOH_SEGMENT_DEFAULT_SIZE = 4 * 1024 * 1024;
+        // Size to commit on demand in that reserved space
+        private const nuint FOH_COMMIT_SIZE = 64 * 1024;
+
+        public T? TryAllocateObject<T>() where T : class
+        {
+            MethodTable* pMT = MethodTable.Of<T>();
+            return Unsafe.As<T?>(TryAllocateObject(pMT, pMT->BaseSize));
+        }
+
+        private object? TryAllocateObject(MethodTable* type, nuint objectSize)
+        {
+            HalfBakedObject* obj = null;
+
+            m_Crst.Acquire();
+
+            try
+            {
+                Debug.Assert(type != null);
+                // _ASSERT(FOH_COMMIT_SIZE >= MIN_OBJECT_SIZE);
+
+                // Currently we don't support frozen objects with special alignment requirements
+                // TODO: We should also give up on arrays of doubles on 32-bit platforms.
+                // (we currently never allocate them on frozen segments)
+#if FEATURE_64BIT_ALIGNMENT
+                if (type->RequiresAlign8)
+                {
+                    // Align8 objects are not supported yet
+                    return nullptr;
+                }
+#endif
+
+                // NOTE: objectSize is expected be the full size including header
+                // _ASSERT(objectSize >= MIN_OBJECT_SIZE);
+
+                if (objectSize > FOH_COMMIT_SIZE)
+                {
+                    // The current design doesn't allow objects larger than FOH_COMMIT_SIZE and
+                    // since FrozenObjectHeap is just an optimization, let's not fill it with huge objects.
+                    return null;
+                }
+
+                obj = m_CurrentSegment == null ? null : m_CurrentSegment.TryAllocateObject(type, objectSize);
+                // obj is nullptr if the current segment is full or hasn't been allocated yet
+                if (obj == null)
+                {
+                    nuint newSegmentSize = FOH_SEGMENT_DEFAULT_SIZE;
+                    if (m_CurrentSegment != null)
+                    {
+                        // Double the reserved size to reduce the number of frozen segments in apps with lots of frozen objects
+                        // Use the same size in case if prevSegmentSize*2 operation overflows.
+                        nuint prevSegmentSize = m_CurrentSegment.m_Size;
+                        newSegmentSize = Math.Max(prevSegmentSize, prevSegmentSize * 2);
+                    }
+
+                    m_CurrentSegment = new FrozenObjectSegment(newSegmentSize);
+
+                    // Try again
+                    obj = m_CurrentSegment.TryAllocateObject(type, objectSize);
+
+                    // This time it's not expected to be null
+                    Debug.Assert(obj != null);
+                }
+            } // end of m_Crst lock
+            finally
+            {
+                m_Crst.Release();
+            }
+
+            IntPtr result = (IntPtr)obj;
+
+            return Unsafe.As<IntPtr, object>(ref result);
+        }
+
+        private class FrozenObjectSegment
+        {
+            // Start of the reserved memory, the first object starts at "m_pStart + sizeof(ObjHeader)" (its pMT)
+            private byte* m_pStart;
+
+            // Pointer to the end of the current segment, ready to be used as a pMT for a new object
+            // meaning that "m_pCurrent - sizeof(ObjHeader)" is the actual start of the new object (header).
+            //
+            // m_pCurrent <= m_SizeCommitted
+            public byte* m_pCurrent;
+
+            // Memory committed in the current segment
+            //
+            // m_SizeCommitted <= m_pStart + FOH_SIZE_RESERVED
+            public nuint m_SizeCommitted;
+
+            // Total memory reserved for the current segment
+            public nuint m_Size;
+
+            private IntPtr m_SegmentHandle;
+
+            public FrozenObjectSegment(nuint sizeHint)
+            {
+                m_Size = sizeHint;
+
+                Debug.Assert(m_Size > FOH_COMMIT_SIZE);
+                Debug.Assert(m_Size % FOH_COMMIT_SIZE == 0);
+
+                void* alloc = ClrVirtualReserve(m_Size);
+                if (alloc == null)
+                {
+                    // Try again with the default FOH size
+                    if (m_Size > FOH_SEGMENT_DEFAULT_SIZE)
+                    {
+                        m_Size = FOH_SEGMENT_DEFAULT_SIZE;
+                        Debug.Assert(m_Size > FOH_COMMIT_SIZE);
+                        Debug.Assert(m_Size % FOH_COMMIT_SIZE == 0);
+                        alloc = ClrVirtualReserve(m_Size);
+                    }
+
+                    if (alloc == null)
+                    {
+                        throw new OutOfMemoryException();
+                    }
+                }
+
+                // Commit a chunk in advance
+                m_pStart = (byte*)ClrVirtualCommit(alloc, FOH_COMMIT_SIZE);
+                if (m_pStart == null)
+                {
+                    ClrVirtualFree(alloc, m_Size);
+                    throw new OutOfMemoryException();
+                }
+
+                m_pCurrent = m_pStart + sizeof(ObjHeader);
+
+                m_SegmentHandle = RuntimeImports.RhRegisterFrozenSegment(m_pStart, (nuint)m_pCurrent - (nuint)m_pStart, FOH_COMMIT_SIZE, m_Size);
+                if (m_SegmentHandle == IntPtr.Zero)
+                {
+                    ClrVirtualFree(alloc, m_Size);
+                    throw new OutOfMemoryException();
+                }
+
+                m_SizeCommitted = FOH_COMMIT_SIZE;
+            }
+
+            public HalfBakedObject* TryAllocateObject(MethodTable* type, nuint objectSize)
+            {
+                Debug.Assert((m_pStart != null) && (m_Size > 0));
+                //_ASSERT(IS_ALIGNED(m_pCurrent, DATA_ALIGNMENT));
+                //_ASSERT(IS_ALIGNED(objectSize, DATA_ALIGNMENT));
+                Debug.Assert(objectSize <= FOH_COMMIT_SIZE);
+                Debug.Assert(m_pCurrent >= m_pStart + sizeof(ObjHeader));
+
+                nuint spaceUsed = (nuint)(m_pCurrent - m_pStart);
+                nuint spaceLeft = m_Size - spaceUsed;
+
+                Debug.Assert(spaceUsed >= (nuint)sizeof(ObjHeader));
+                Debug.Assert(spaceLeft >= (nuint)sizeof(ObjHeader));
+
+                // Test if we have a room for the given object (including extra sizeof(ObjHeader) for next object)
+                if (spaceLeft - (nuint)sizeof(ObjHeader) < objectSize)
+                {
+                    return null;
+                }
+
+                // Check if we need to commit a new chunk
+                if (spaceUsed + objectSize + (nuint)sizeof(ObjHeader) > m_SizeCommitted)
+                {
+                    // Make sure we don't go out of bounds during this commit
+                    Debug.Assert(m_SizeCommitted + FOH_COMMIT_SIZE <= m_Size);
+
+                    if (ClrVirtualCommit(m_pStart + m_SizeCommitted, FOH_COMMIT_SIZE) == null)
+                    {
+                        throw new OutOfMemoryException();
+                    }
+                    m_SizeCommitted += FOH_COMMIT_SIZE;
+                }
+
+                HalfBakedObject* obj = (HalfBakedObject*)m_pCurrent;
+                obj->SetMethodTable(type);
+
+                m_pCurrent += objectSize;
+
+                RuntimeImports.RhUpdateFrozenSegment(m_SegmentHandle, m_pCurrent, m_pStart + m_SizeCommitted);
+
+                return obj;
+            }
+        }
+
+        private struct HalfBakedObject
+        {
+            private MethodTable* _methodTable;
+            public void SetMethodTable(MethodTable* methodTable) => _methodTable = methodTable;
+        }
+    }
+}