feat(llvm): try (and fail) to workaround bad modular addition codegen…

… with inline function.

constantine/math_compiler/impl_fields_sat.nim

@@ -86,7 +86,10 @@ proc finalSubMayOverflow*(asy: Assembler_LLVM, fd: FieldDescriptor, r, a, M, car
   ## also overflow the limbs (a 2^256 order of magnitude modulus stored in n words of total max size 2^256)
 
   let name = "_finalsub_mayo_u" & $fd.w & "x" & $fd.numWords
-  asy.llvmInternalFnDef(name, SectionName, asy.void_t, toTypes([r, a, M, carry])):
+  asy.llvmInternalFnDef(
+          name, SectionName,
+          asy.void_t, toTypes([r, a, M, carry]),
+          {kHot, kInline}):
 
     let (rr, aa, MM, carry) = llvmParams
 
@@ -126,7 +129,10 @@ proc finalSubNoOverflow*(asy: Assembler_LLVM, fd: FieldDescriptor, r, a, M: Valu
   ## (say using 255 bits for the modulus out of 256 available in words)
 
   let name = "_finalsub_noo_u" & $fd.w & "x" & $fd.numWords
-  asy.llvmInternalFnDef(name, SectionName, asy.void_t, toTypes([r, a, M])):
+  asy.llvmInternalFnDef(
+          name, SectionName,
+          asy.void_t, toTypes([r, a, M]),
+          {kHot, kInline}):
 
     let (rr, aa, MM) = llvmParams
 
@@ -156,7 +162,10 @@ proc modadd*(asy: Assembler_LLVM, fd: FieldDescriptor, r, a, b, M: ValueRef) =
   let red = if fd.spareBits >= 1: "noo"
             else: "mayo"
   let name = "_modadd_" & red & "_u" & $fd.w & "x" & $fd.numWords
-  asy.llvmInternalFnDef(name, SectionName, asy.void_t, toTypes([r, a, b, M])):
+  asy.llvmInternalFnDef(
+          name, SectionName,
+          asy.void_t, toTypes([r, a, b, M]),
+          {kHot}):
 
     let (r, aa, bb, M) = llvmParams
 

constantine/math_compiler/ir.nim

@@ -18,6 +18,16 @@ import
 # ############################################################
 
 type
+  AttrKind* = enum
+    # Other important properties like
+    # - norecurse
+    # - memory side-effects memory(argmem: readwrtite)
+    #   can be deduced.
+    kHot,
+    kInline,
+    kAlwaysInline,
+    kNoInline
+
   Assembler_LLVM* = ref object
     ctx*: ContextRef
     module*: ModuleRef
@@ -31,7 +41,9 @@ type
     # It doesn't seem possible to retrieve a function type
     # from its value, so we store them here.
     # If we store the type we might as well store the impl
-    fns: Table[string, tuple[ty: TypeRef, impl: ValueRef]]
+    # and we store whether it's internal to apply the fastcc calling convention
+    fns: Table[string, tuple[ty: TypeRef, impl: ValueRef, internal: bool]]
+    attrs: array[AttrKind, AttributeRef]
 
     # Convenience
     void_t*: TypeRef
@@ -96,6 +108,11 @@ proc new*(T: type Assembler_LLVM, backend: Backend, moduleName: cstring): Assemb
 
   result.configure(backend)
 
+  result.attrs[kHot] = result.ctx.createAttr("hot")
+  result.attrs[kInline] = result.ctx.createAttr("inlinehint")
+  result.attrs[kAlwaysInline] = result.ctx.createAttr("alwaysinline")
+  result.attrs[kNoInline] = result.ctx.createAttr("noinline")
+
 # ############################################################
 #
 #                     Syntax Sugar
@@ -333,7 +350,7 @@ proc tagCudaKernel(asy: Assembler_LLVM, fn: ValueRef) =
 
 proc setPublic(asy: Assembler_LLVM, fn: ValueRef) =
   case asy.backend
-  of bkAmdGpu: fn.setCallingConvention(AMDGPU_KERNEL)
+  of bkAmdGpu: fn.setFnCallConv(AMDGPU_KERNEL)
   of bkNvidiaPtx: asy.tagCudaKernel(fn)
   else: discard
 
@@ -432,16 +449,22 @@ macro unpackParams[N: static int](
       let fn = `br`.getCurrentFunction()
       fn.getParam(uint32 `i`)
 
+proc addAttributes(asy: Assembler_LLVM, fn: ValueRef, attrs: set[AttrKind]) =
+  for attr in attrs:
+    fn.addAttribute(kAttrFnIndex, asy.attrs[attr])
+
+  fn.addAttribute(kAttrFnIndex, asy.attrs[kHot])
+
 template llvmFnDef[N: static int](
           asy: Assembler_LLVM,
           name, sectionName: string,
           returnType: TypeRef,
           paramTypes: array[N, TypeRef],
           internal: bool,
+          attrs: set[AttrKind],
           body: untyped) =
   ## This setups common prologue to implement a function in LLVM
   ## Function parameters are available with the `llvmParams` magic variable
-
   let paramsTys = asy.wrapTypesForFnCall(paramTypes)
 
   var fn = asy.module.getFunction(cstring name)
@@ -451,7 +474,7 @@ template llvmFnDef[N: static int](
     let fnTy = function_t(returnType, paramsTys.wrapped)
     fn = asy.module.addFunction(cstring name, fnTy)
 
-    asy.fns[name] = (fnTy, fn)
+    asy.fns[name] = (fnTy, fn, internal)
 
     let blck = asy.ctx.appendBasicBlock(fn)
     asy.br.positionAtEnd(blck)
@@ -465,11 +488,12 @@ template llvmFnDef[N: static int](
     body
 
     if internal:
-      fn.setCallingConvention(Fast)
+      fn.setFnCallConv(Fast)
       fn.setLinkage(linkInternal)
     else:
       asy.setPublic(fn)
     fn.setSection(sectionName)
+    asy.addAttributes(fn, attrs)
 
     asy.br.positionAtEnd(savedLoc)
 
@@ -478,26 +502,30 @@ template llvmInternalFnDef*[N: static int](
           name, sectionName: string,
           returnType: TypeRef,
           paramTypes: array[N, TypeRef],
+          attrs: set[AttrKind] = {},
           body: untyped) =
-  llvmFnDef(asy, name, sectionName, returnType, paramTypes, internal = true, body)
+  llvmFnDef(asy, name, sectionName, returnType, paramTypes, internal = true, attrs, body)
 
 template llvmPublicFnDef*[N: static int](
           asy: Assembler_LLVM,
           name, sectionName: string,
           returnType: TypeRef,
           paramTypes: array[N, TypeRef],
           body: untyped) =
-  llvmFnDef(asy, name, sectionName, returnType, paramTypes, internal = false, body)
+  llvmFnDef(asy, name, sectionName, returnType, paramTypes, internal = false, {}, body)
 
 proc callFn*(
       asy: Assembler_LLVM,
       name: string,
       params: openArray[ValueRef]): ValueRef {.discardable.} =
 
   if asy.fns[name].ty.getReturnType().getTypeKind() == tkVoid:
-    asy.br.call2(asy.fns[name].ty, asy.fns[name].impl, params)
+    result = asy.br.call2(asy.fns[name].ty, asy.fns[name].impl, params)
   else:
-    asy.br.call2(asy.fns[name].ty, asy.fns[name].impl, params, cstring(name))
+    result = asy.br.call2(asy.fns[name].ty, asy.fns[name].impl, params, cstring(name))
+
+  if asy.fns[name].internal:
+    result.setInstrCallConv(Fast)
 
 # ############################################################
 #

constantine/platforms/abis/llvm_abi.nim

@@ -45,6 +45,7 @@ type
   TypeRef* = distinct pointer
   ValueRef* = distinct pointer
   MetadataRef = distinct pointer
+  AttributeRef* = distinct pointer
   LLVMstring = distinct cstring
   ErrorMessageString = distinct cstring
     ## A string with a buffer owned by LLVM
@@ -123,16 +124,19 @@ proc verify(module: ModuleRef, failureAction: VerifierFailureAction, msg: var LL
 
 {.push used.}
 proc initializeX86AsmPrinter() {.importc: "LLVMInitializeX86AsmPrinter".}
+proc initializeX86AsmParser() {.importc: "LLVMInitializeX86AsmParser".}
 proc initializeX86Target() {.importc: "LLVMInitializeX86Target".}
 proc initializeX86TargetInfo() {.importc: "LLVMInitializeX86TargetInfo".}
 proc initializeX86TargetMC() {.importc: "LLVMInitializeX86TargetMC".}
 
 proc initializeNVPTXAsmPrinter() {.importc: "LLVMInitializeNVPTXAsmPrinter".}
+proc initializeNVPTXAsmParser() {.importc: "LLVMInitializeNVPTXAsmParser".}
 proc initializeNVPTXTarget() {.importc: "LLVMInitializeNVPTXTarget".}
 proc initializeNVPTXTargetInfo() {.importc: "LLVMInitializeNVPTXTargetInfo".}
 proc initializeNVPTXTargetMC() {.importc: "LLVMInitializeNVPTXTargetMC".}
 
 proc initializeAMDGPUAsmPrinter() {.importc: "LLVMInitializeAMDGPUAsmPrinter".}
+proc initializeAMDGPUAsmParser() {.importc: "LLVMInitializeAMDGPUAsmParser".}
 proc initializeAMDGPUTarget() {.importc: "LLVMInitializeAMDGPUTarget".}
 proc initializeAMDGPUTargetInfo() {.importc: "LLVMInitializeAMDGPUTargetInfo".}
 proc initializeAMDGPUTargetMC() {.importc: "LLVMInitializeAMDGPUTargetMC".}
@@ -612,8 +616,24 @@ proc countParamTypes*(functionTy: TypeRef): uint32 {.importc: "LLVMCountParamTyp
 
 proc getCalledFunctionType*(fn: ValueRef): TypeRef {.importc: "LLVMGetCalledFunctionType".}
 
-proc getCallingConvention*(function: ValueRef): CallingConvention {.importc: "LLVMGetFunctionCallConv".}
-proc setCallingConvention*(function: ValueRef, cc: CallingConvention) {.importc: "LLVMSetFunctionCallConv".}
+proc getFnCallConv*(function: ValueRef): CallingConvention {.importc: "LLVMGetFunctionCallConv".}
+proc setFnCallConv*(function: ValueRef, cc: CallingConvention) {.importc: "LLVMSetFunctionCallConv".}
+
+proc getInstrCallConv*(instr: ValueRef): CallingConvention {.importc: "LLVMGetInstructionCallConv".}
+proc setInstrCallConv*(instr: ValueRef, cc: CallingConvention) {.importc: "LLVMSetInstructionCallConv".}
+
+type
+  AttributeIndex* {.size: sizeof(cint).} = enum
+    ## Attribute index is either -1 for the function
+    ## 0 for the return value
+    ## or 1..n for each function parameter
+    kAttrFnIndex = -1
+    kAttrRetIndex = 0
+
+proc toAttrId*(name: openArray[char]): cuint {.importc: "LLVMGetEnumAttributeKindForName".}
+proc toAttr*(ctx: ContextRef, attr_id: uint64, val = 0'u64): AttributeRef {.importc: "LLVMCreateEnumAttribute".}
+proc addAttribute*(fn: ValueRef, index: cint, attr: AttributeRef) {.importc: "LLVMAddAttributeAtIndex".}
+proc addAttribute*(fn: ValueRef, index: AttributeIndex, attr: AttributeRef) {.importc: "LLVMAddAttributeAtIndex".}
 
 # ############################################################
 #

constantine/platforms/llvm/llvm.nim

@@ -175,6 +175,12 @@ proc array_t*(elemType: TypeRef, elemCount: SomeInteger): TypeRef {.inline.}=
 proc function_t*(returnType: TypeRef, paramTypes: openArray[TypeRef]): TypeRef {.inline.} =
   function_t(returnType, paramTypes, isVarArg = LlvmBool(false))
 
+# Functions
+# ------------------------------------------------------------
+
+proc createAttr*(ctx: ContextRef, name: openArray[char]): AttributeRef =
+  ctx.toAttr(name.toAttrId())
+
 # Values
 # ------------------------------------------------------------
 
@@ -189,4 +195,3 @@ proc getName*(v: ValueRef): string =
 
 proc constInt*(ty: TypeRef, n: SomeInteger, signExtend = false): ValueRef {.inline.} =
   constInt(ty, culonglong(n), LlvmBool(signExtend))
-

research/codegen/x86_poc.nim

@@ -65,16 +65,17 @@ proc t_field_add() =
     target = toTarget(triple),
     triple = triple,
     cpu = "",
-    features = "adx,bmi2", # TODO check the proper way to pass options
-    level = CodeGenLevelAggressive,
+    features = "", # "adx,bmi2", # TODO check the proper way to pass options
+    level = CodeGenLevelDefault,
     reloc = RelocDefault,
     codeModel = CodeModelDefault
   )
 
   let pbo = createPassBuilderOptions()
   pbo.setMergeFunctions()
   let err = asy.module.runPasses(
-    "default<O3>,function-attrs,memcpyopt,sroa,mem2reg,gvn,dse,instcombine,inline,adce",
+    "default<O2>",
+    # "default<O2>,memcpyopt,sroa,mem2reg,function-attrs,inline,gvn,dse,aggressive-instcombine,adce",
     machine,
     pbo
   )