Fix for #89645, stack overflow in Crossgen2

src/coreclr/tools/Common/Compiler/CompilerTypeSystemContext.Validation.cs

@@ -18,6 +18,13 @@ public partial class CompilerTypeSystemContext
         private static List<TypeDesc> EmptyList = new List<TypeDesc>();
         private readonly ValidTypeHashTable _validTypes = new ValidTypeHashTable();
 
+        /// <summary>
+        /// Once the type check stack is this deep, we declare the type being scanned as
+        /// recursive. In practice, for recursive types, stack overflow happens when the type
+        /// load stack is approximately twice as deep (1800~1900).
+        /// </summary>
+        private const int MaximumTypeLoadCheckStackDepth = 1024;
+
         /// <summary>
         /// Ensures that the type can be fully loaded. The method will throw one of the type system
         /// exceptions if the type is not loadable.
@@ -103,6 +110,12 @@ private static bool PushTypeLoadInProgress(TypeDesc type)
         {
             t_typeLoadCheckInProgressStack ??= new List<TypeLoadabilityCheckInProgress>();
 
+            if (t_typeLoadCheckInProgressStack.Count >= MaximumTypeLoadCheckStackDepth)
+            {
+                // Extreme stack depth typically indicates infinite recursion in recursive descent into the type
+                ThrowHelper.ThrowTypeLoadException(ExceptionStringID.ClassLoadGeneral, type);
+            }
+
             // Walk stack to see if the specified type is already in the process of being type checked.
             int typeLoadCheckInProgressStackOffset = -1;
             bool checkingMode = false; // Checking for match on TypeLoadabilityCheck field or in OtherTypesToMarkAsSuccessfullyLoaded. (true for OtherTypesToMarkAsSuccessfullyLoaded)