Implement dispatch codegen

Handler frequency statistics, collected via SPMI:

benchmarks.pmi

Total clauses       : 2514
Finally             : 281    11.2%
Fault               : 1413   56.2%
Single catch        : 724    28.8%
Single filter       : 49      1.9%
Mutually protecting : 47      1.9%
Among them, catches : 108    86.4%
Among them, filters : 17     13.6%

libraries.pmi

Total clauses       : 19568
Finally             : 1937    9.9%
Fault               : 11399  58.3%
Single catch        : 5373   27.5%
Single filter       : 349     1.8%
Mutually protecting : 510     2.6%
Among them, catches : 1240   95.0%
Among them, filters : 65      5.0%

Passing (with RyuJit codegen) tests:

 - baseservices\exceptions\simple\finally.csproj
 - baseservices\exceptions\simple\fault.ilproj
 - baseservices\exceptions\unittests\baseclass.csproj
 - baseservices\exceptions\unittests\Baadbaad.csproj
 - baseservices\exceptions\unittests\TryCatchInFinallyTest.csproj
 - baseservices\exceptions\unittests\ThrowInFinallyNestedInTry.csproj
 - baseservices\exceptions\unittests\ThrowInFinallyTest.csproj
 - baseservices\exceptions\unittests\ThrowInCatch.csproj
 - baseservices\exceptions\unittests\ReturnFromCatch.csproj
 - baseservices\exceptions\unittests\RethrowAndFinally.csproj
 - baseservices\exceptions\unittests\RecursiveThrowNew.csproj
 - baseservices\exceptions\unittests\RecursiveRethrow.csproj
 - baseservices\exceptions\unittests\Recurse.csproj
 - baseservices\exceptions\unittests\Pending.csproj
 - baseservices\exceptions\unittests\InnerFinallyAndCatch.csproj
 - baseservices\exceptions\unittests\InnerFinally.csproj
 - baseservices\exceptions\unittests\GoryNativePast.csproj
 - baseservices\exceptions\unittests\GoryManagedPresent.csproj
 - baseservices\exceptions\unittests\CollidedUnwind.csproj

src/coreclr/jit/compiler.h

@@ -4337,6 +4337,8 @@ class Compiler
                              bool                    isExplicitTailCall,
                              IL_OFFSET               ilOffset = BAD_IL_OFFSET);
 
+    void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind);
+
     //=========================================================================
     //                          PROTECTED
     //=========================================================================
@@ -4357,8 +4359,6 @@ class Compiler
     bool impIsPrimitive(CorInfoType type);
     bool impILConsumesAddr(const BYTE* codeAddr);
 
-    void impResolveToken(const BYTE* addr, CORINFO_RESOLVED_TOKEN* pResolvedToken, CorInfoTokenKind kind);
-
     void impPushOnStack(GenTree* tree, typeInfo ti);
     void        impPushNullObjRefOnStack();
     StackEntry  impPopStack();
@@ -6555,6 +6555,8 @@ class Compiler
 
     AddCodeDsc* fgFindExcptnTarget(SpecialCodeKind kind, unsigned refData);
 
+    bool fgIsThrowHlpBlk(BasicBlock* block);
+
     bool fgUseThrowHelperBlocks();
 
     AddCodeDsc* fgGetAdditionalCodeDescriptors()
@@ -6565,8 +6567,6 @@ class Compiler
 private:
     bool fgIsCodeAdded();
 
-    bool fgIsThrowHlpBlk(BasicBlock* block);
-
 #if !FEATURE_FIXED_OUT_ARGS
     unsigned fgThrowHlpBlkStkLevel(BasicBlock* block);
 #endif // !FEATURE_FIXED_OUT_ARGS

src/coreclr/jit/compiler.hpp

@@ -2705,7 +2705,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block)
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWDIVZERO)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTEXCEPTION)) ||
           (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROW_ARGUMENTOUTOFRANGEEXCEPTION)) ||
-          (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW))))
+          (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_OVERFLOW)) ||
+          (call->AsCall()->gtCallMethHnd == eeFindHelper(CORINFO_HELP_THROWNULLREF))))
     {
         return false;
     }
@@ -2719,7 +2720,8 @@ inline bool Compiler::fgIsThrowHlpBlk(BasicBlock* block)
         if (block == add->acdDstBlk)
         {
             return add->acdKind == SCK_RNGCHK_FAIL || add->acdKind == SCK_DIV_BY_ZERO || add->acdKind == SCK_OVERFLOW ||
-                   add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN;
+                   add->acdKind == SCK_ARG_EXCPN || add->acdKind == SCK_ARG_RNG_EXCPN ||
+                   add->acdKind == SCK_NULL_REF_EXCPN;
         }
     }
 

src/coreclr/jit/fgdiagnostic.cpp

@@ -1660,6 +1660,8 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos)
                         case EH_HANDLER_FAULT_WAS_FINALLY:
                             handlerType = "fault-was-finally";
                             break;
+                        default:
+                            unreached();
                     }
                     sprintf_s(name, sizeof(name), "EH#%u %s", XTnum, handlerType);
                     rgnGraph.Insert(name, RegionGraph::RegionType::EH, ehDsc->ebdHndBeg, ehDsc->ebdHndLast);

src/coreclr/jit/jiteh.h

@@ -27,7 +27,8 @@ enum EHHandlerType
     EH_HANDLER_FILTER,
     EH_HANDLER_FAULT,
     EH_HANDLER_FINALLY,
-    EH_HANDLER_FAULT_WAS_FINALLY
+    EH_HANDLER_FAULT_WAS_FINALLY,
+    EH_HANDLER_COUNT
 };
 
 // ToCORINFO_EH_CLAUSE_FLAGS: Convert an internal EHHandlerType to a CORINFO_EH_CLAUSE_FLAGS value

src/coreclr/jit/llvm.cpp

@@ -23,6 +23,7 @@ enum class EEApiId
 {
     GetMangledMethodName,
     GetSymbolMangledName,
+    GetEHDispatchFunctionName,
     GetTypeName,
     AddCodeReloc,
     IsRuntimeImport,
@@ -104,7 +105,6 @@ Llvm::Llvm(Compiler* compiler)
     _info(compiler->info),
     _sigInfo(compiler->info.compMethodInfo->args),
     _builder(_llvmContext),
-    _prologBuilder(_llvmContext),
     _blkToLlvmBlksMap(compiler->getAllocator(CMK_Codegen)),
     _sdsuMap(compiler->getAllocator(CMK_Codegen)),
     _localsMap(compiler->getAllocator(CMK_Codegen)),
@@ -169,6 +169,7 @@ GCInfo* Llvm::getGCInfo()
     {
         _gcInfo = new (_compiler->getAllocator(CMK_GC)) GCInfo(_compiler);
     }
+
     return _gcInfo;
 }
 
@@ -377,8 +378,8 @@ const HelperFuncInfo& Llvm::getHelperFuncInfo(CorInfoHelpFunc helperFunc)
         // For WASM, currently implemented in the bootstrapper...
         { FUNC(CORINFO_HELP_THROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } },
 
-        // (Not) implemented in "Runtime\EHHelpers.cpp"
-        { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { } },
+        // Implemented in "Runtime.Base\src\System\Runtime\ExceptionHandling.wasm.cs".
+        { FUNC(CORINFO_HELP_RETHROW) CORINFO_TYPE_VOID, { CORINFO_TYPE_PTR } },
 
         // Implemented in "Runtime\MiscHelpers.cpp".
         { FUNC(CORINFO_HELP_USER_BREAKPOINT) CORINFO_TYPE_VOID, { } },
@@ -654,11 +655,11 @@ unsigned Llvm::padNextOffset(CorInfoType corInfoType, CORINFO_CLASS_HANDLE struc
 
 [[noreturn]] void Llvm::failFunctionCompilation()
 {
-    for (Function* llvmFunc : m_functions)
+    for (FunctionInfo& funcInfo : m_functions)
     {
-        if (llvmFunc != nullptr)
+        if (funcInfo.LlvmFunction != nullptr)
         {
-            llvmFunc->deleteBody();
+            funcInfo.LlvmFunction->deleteBody();
         }
     }
 
@@ -681,6 +682,11 @@ const char* Llvm::GetMangledSymbolName(void* symbol)
     return CallEEApi<EEApiId::GetSymbolMangledName, const char*>(symbol);
 }
 
+const char* Llvm::GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType)
+{
+    return CallEEApi<EEApiId::GetEHDispatchFunctionName, const char*>(handlerType);
+}
+
 const char* Llvm::GetTypeName(CORINFO_CLASS_HANDLE typeHandle)
 {
     return CallEEApi<EEApiId::GetTypeName, const char*>(typeHandle);

src/coreclr/jit/llvm.h

@@ -109,6 +109,17 @@ struct LlvmBlockRange
     }
 };
 
+typedef JitHashTable<unsigned, JitSmallPrimitiveKeyFuncs<unsigned>, llvm::AllocaInst*> AllocaMap;
+
+struct FunctionInfo
+{
+    Function* LlvmFunction;
+    union {
+        llvm::AllocaInst** Allocas; // Dense "lclNum -> Alloca*" mapping used for the main function.
+        AllocaMap* AllocaMap; // Sparse "lclNum -> Alloca*" mapping used for funclets.
+    };
+};
+
 // TODO: We should create a Static... class to manage the globals and their lifetimes.
 // Note we declare all statics here, and define them in llvm.cpp, for documentation and
 // visibility purposes even as some are only needed in other compilation units.
@@ -132,16 +143,15 @@ class Llvm
     BasicBlock* m_currentBlock;
     DebugInfo _currentOffset;
     llvm::IRBuilder<> _builder;
-    llvm::IRBuilder<> _prologBuilder;
     JitHashTable<BasicBlock*, JitPtrKeyFuncs<BasicBlock>, LlvmBlockRange> _blkToLlvmBlksMap;
     JitHashTable<GenTree*, JitPtrKeyFuncs<GenTree>, Value*> _sdsuMap;
     JitHashTable<SSAName, SSAName, Value*> _localsMap;
     std::vector<PhiPair> _phiPairs;
-    std::vector<Value*> m_allocas;
-    std::vector<Function*> m_functions;
+    std::vector<FunctionInfo> m_functions;
     std::vector<llvm::BasicBlock*> m_EHDispatchLlvmBlocks;
 
     // Codegen emit context.
+    unsigned m_currentLlvmFunctionIndex = ROOT_FUNC_IDX;
     unsigned m_currentProtectedRegionIndex = EHblkDsc::NO_ENCLOSING_INDEX;
     LlvmBlockRange* m_currentLlvmBlocks = nullptr;
 
@@ -198,6 +208,7 @@ class Llvm
     //
     const char* GetMangledMethodName(CORINFO_METHOD_HANDLE methodHandle);
     const char* GetMangledSymbolName(void* symbol);
+    const char* GetEHDispatchFunctionName(CORINFO_EH_CLAUSE_FLAGS handlerType);
     const char* GetTypeName(CORINFO_CLASS_HANDLE typeHandle);
     void AddCodeReloc(void* handle);
     bool IsRuntimeImport(CORINFO_METHOD_HANDLE methodHandle);
@@ -223,6 +234,7 @@ class Llvm
     Type* getLlvmTypeForLclVar(LclVarDsc* varDsc);
     Type* getLlvmTypeForCorInfoType(CorInfoType corInfoType, CORINFO_CLASS_HANDLE classHnd);
     Type* getLlvmTypeForParameterType(CORINFO_CLASS_HANDLE classHnd);
+    Type* getIntPtrLlvmType();
 
     unsigned getElementSize(CORINFO_CLASS_HANDLE fieldClassHandle, CorInfoType corInfoType);
     void addPaddingFields(unsigned paddingSize, std::vector<Type*>& llvmFields);
@@ -247,6 +259,8 @@ class Llvm
     void lowerFieldOfDependentlyPromotedStruct(GenTree* node);
     void ConvertShadowStackLocalNode(GenTreeLclVarCommon* node);
     void lowerCall(GenTreeCall* callNode);
+    void lowerRethrow(GenTreeCall* callNode);
+    void lowerCatchArg(GenTree* catchArgNode);
     void lowerIndir(GenTreeIndir* indirNode);
     void lowerStoreBlk(GenTreeBlk* storeBlkNode);
     void lowerDivMod(GenTreeOp* divModNode);
@@ -265,6 +279,11 @@ class Llvm
     bool isShadowFrameLocal(LclVarDsc* varDsc) const;
     bool isFuncletParameter(unsigned lclNum) const;
 
+    unsigned getCurrentShadowFrameSize() const;
+    unsigned getShadowFrameSize(unsigned hndIndex) const;
+    unsigned getOriginalShadowFrameSize() const;
+    unsigned getCatchArgOffset() const;
+
     // ================================================================================================================
     // |                                                   Codegen                                                    |
     // ================================================================================================================
@@ -273,14 +292,14 @@ class Llvm
     void Compile();
 
 private:
-    const int ROOT_FUNC_IDX = 0;
+    const unsigned ROOT_FUNC_IDX = 0;
 
     bool initializeFunctions();
     void generateProlog();
     void initializeLocals();
-    void generateFuncletProlog(unsigned funcIdx);
     void generateBlock(BasicBlock* block);
     void generateEHDispatch();
+    Value* generateEHDispatchTable(Function* llvmFunc, unsigned innerEHIndex, unsigned outerEHIndex);
     void fillPhis();
 
     Value* getGenTreeValue(GenTree* node);
@@ -313,13 +332,11 @@ class Llvm
     void buildBinaryOperation(GenTree* node);
     void buildShift(GenTreeOp* node);
     void buildReturn(GenTree* node);
-    void buildCatchArg(GenTree* node);
     void buildJTrue(GenTree* node);
     void buildNullCheck(GenTreeIndir* nullCheckNode);
     void buildBoundsCheck(GenTreeBoundsChk* boundsCheck);
 
     void buildCallFinally(BasicBlock* block);
-    void buildCatchReturn(BasicBlock* block);
 
     void storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc);
     unsigned buildMemCpy(Value* baseAddress, unsigned startOffset, unsigned endOffset, Value* srcAddress);
@@ -328,14 +345,17 @@ class Llvm
     void emitJumpToThrowHelper(Value* jumpCondValue, SpecialCodeKind throwKind);
     void emitNullCheckForIndir(GenTreeIndir* indir, Value* addrValue);
     Value* emitHelperCall(CorInfoHelpFunc helperFunc, ArrayRef<Value*> sigArgs = { });
-    Value* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef<Value*> args);
+    llvm::CallBase* emitCallOrInvoke(llvm::FunctionCallee callee, ArrayRef<Value*> args);
 
     FunctionType* getFunctionType();
     Function* getOrCreateLlvmFunction(const char* symbolName, GenTreeCall* call);
     FunctionType* createFunctionTypeForCall(GenTreeCall* call);
     FunctionType* createFunctionTypeForHelper(CorInfoHelpFunc helperFunc);
 
-    Value* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr);
+    llvm::GlobalVariable* getOrCreateExternalSymbol(const char* symbolName, Type* symbolType = nullptr);
+    llvm::GlobalVariable* getOrCreateSymbol(CORINFO_GENERIC_HANDLE symbolHandle, Type* symbolType = nullptr);
+    Value* emitSymbolRef(CORINFO_GENERIC_HANDLE symbolHandle);
+    CORINFO_GENERIC_HANDLE getSymbolHandleForClassToken(mdToken token);
 
     Instruction* getCast(Value* source, Type* targetType);
     Value* castIfNecessary(Value* source, Type* targetType, llvm::IRBuilder<>* builder = nullptr);
@@ -346,27 +366,32 @@ class Llvm
 
     DebugMetadata getOrCreateDebugMetadata(const char* documentFileName);
     llvm::DILocation* createDebugFunctionAndDiLocation(struct DebugMetadata debugMetadata, unsigned int lineNo);
+    llvm::DILocation* getArtificialDebugLocation();
 
     void setCurrentEmitContextForBlock(BasicBlock* block);
-    void setCurrentEmitContext(LlvmBlockRange* llvmBlock, unsigned tryIndex);
-    LlvmBlockRange* getCurrentLlvmBlocks() const;
+    void setCurrentEmitContext(unsigned funcIdx, unsigned tryIndex, LlvmBlockRange* llvmBlock);
+    unsigned getCurrentLlvmFunctionIndex() const;
     unsigned getCurrentProtectedRegionIndex() const;
+    LlvmBlockRange* getCurrentLlvmBlocks() const;
 
     Function* getRootLlvmFunction();
     Function* getCurrentLlvmFunction();
     Function* getLlvmFunctionForIndex(unsigned funcIdx);
-    unsigned getLlvmFunctionIndexForBlock(BasicBlock* block);
-    unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex);
+    FunctionInfo& getLlvmFunctionInfoForIndex(unsigned funcIdx);
+    unsigned getLlvmFunctionIndexForBlock(BasicBlock* block) const;
+    unsigned getLlvmFunctionIndexForProtectedRegion(unsigned tryIndex) const;
 
     llvm::BasicBlock* createInlineLlvmBlock();
     llvm::BasicBlock* getCurrentLlvmBlock() const;
     LlvmBlockRange* getLlvmBlocksForBlock(BasicBlock* block);
     llvm::BasicBlock* getFirstLlvmBlockForBlock(BasicBlock* block);
     llvm::BasicBlock* getLastLlvmBlockForBlock(BasicBlock* block);
+    llvm::BasicBlock* getOrCreatePrologLlvmBlockForFunction(unsigned funcIdx);
+
+    bool isReachable(BasicBlock* block) const;
+    BasicBlock* getFirstBlockForFunction(unsigned funcIdx) const;
 
     Value* getLocalAddr(unsigned lclNum);
-    unsigned getTotalLocalOffset();
+    Value* getOrCreateAllocaForLocalInFunclet(unsigned lclNum);
 };
-
-
 #endif /* End of _LLVM_H_ */