Update llpc from commit 4bd41bcf

Set SLC=0 for ATM
LowerGpuRt: fix a type confusion
Honor NonUniform decorations on OpAccessChain operands
Force WGP when NGG is in passthrough mode
Use idxen modifier for (RW)StructuredBuffers
Compatible with LLVM upstream change that moved Utility funcs to Utils
Include Line Correlation Info for Tools via LLPC
[Continuations] Cleanup Complete op lowering
[CompilerUtils] Improve handling of freeze in ValueOriginTracker
[compilerutils] Add GetGlobalInModule to CrossModuleInliner
amdllpc: some usability improvements
[Continuations] Add SpecializeDriverShadersPass
Split up gl_in array type.
[Continuations] Remove stack lowering define
Use inbounds gep to index row major matrix
llvmraytracing: Remove support for _AmdContinuationsGetFlags
Properly encapsulate state that affects GPURT specialization
llvmraytracing: only support a waitmask of -1
llvmraytracing: remove remaining traces of EnqueueCall
compilerutils: fix a warning
[LowerGlobals] Mark globals with `buffer.index` users as readonly
Implement structural GEP dialect for in/out
llvmraytracing: add LLPC/LLVM scratch/global address spaces
llpc/ProcessGpuRtLibrary: use earlyGpurtTransform
[Continuations] Make use of `llvm::zip` in `CleanupContinuationsPass::updateCpsFunctionArgs`
Adjust the llvm-tblgen memory limit
lgc: split lowering of GroupMemcpyOp for mesh/task shaders to MeshTaskShader
PatchResourceCollect: stop cleaning undef output value in some cases
[Continuations] Freeze poison that is stored to payload
Fix wrong pipeline dump message
Fix issues with coherent
Update SPIR-V header to latest version
Simplify load of attribute ring buffer descriptor
[Continuations] Enable Traversal specialization test
Adjust tests to take into account upstream overload of rsrc and samp arguments
[Continuations] Fix signed/unsigned comparison warning
LowerRaytracingPipeline: tease apart return handling and any-hit exit handling
LowerRaytracingPipeline: unify function end handling
lgc: support int4 in cooperative matrix
Add rounding mode for PackHalf2x16
[Continuations] Fix unused variable warning
lgc: Add MsgPackScanner
[Continuations] Remove `LegacyCleanupContinuations` pass
LowerCooperativeMatrix: fix compile warning due to missing break statement
lgc: New RegStackUsage to propagate reg/stack usage
[LGC] Add helper lane state for subgroups
Add client name to SPIR-V frontend
Use more flexible readfirstlane
Gate argument definition for `GEP::collectOffset`.
[Continuations] Introduce dummy `csp` argument for `lgc.cps.jump`
[Continuations] Tolerate non-waiting AwaitTraversal
Add AmdExtD3DShaderIntrinsics_LoadDwordAtAddrx3
Add more BuiltIn which has primitive index in mesh shader
Rewrite the helper getShaderModuleUsageInfo
[Continuations] Replace isLgcRtOp helper with isDialectOp helper from dialects
Downgrade scope Device to Workgroup if permitted
Optimize performance for PrimSetup
Update submodule llvm-dialects
Remove inactive built-ins for last vertex processing stage
Fix internal hash for color export shader
Fix call to lookupIntrinsicID
[Continuations] Move Simplifying GEP helpers to CompilerUtils
[PatchBufferOp] Generate struct buffer cmpxchg intrinsics
[LGC] Refactor default wave size setting
[Continuations] Derive `DispatchSystemData` type from `_cont_DispatchRaysIndex3`
[Continuations] Replace "lgc.rt" starts_with check
Rename some classes and files
Optimize PointSize write when the value is 1.0
[Continuations] Handle `_AmdGetShaderRecordIndex` calls
[LowerBufferOperations] Check for uniform buffer pointers for s_buffer_load
lgc: Mark applicable LgcDialect and Builder ops as NoDivergenceSource

CMakeLists.txt

@@ -206,7 +206,11 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
         if (TARGET llvm-dialects-example)
             set_property(TARGET llvm-dialects-example PROPERTY FOLDER Misc)
         endif()
-        if (LLVM_OPTIMIZED_TABLEGEN)
+        if (NOT WIN32 AND LLVM_OPTIMIZED_TABLEGEN)
+#if _WIN32
+            # These targets don't exist on Windows when CMake is first invoked.
+            # They are created later at build time, when the cross-compilation takes place.
+#endif
             set_property(TARGET llvm_nm_target PROPERTY FOLDER Misc)
             set_property(TARGET llvm_readobj_target PROPERTY FOLDER Misc)
             set_property(TARGET llvm-min-tblgen-host PROPERTY FOLDER Misc)
@@ -215,9 +219,8 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
             set_property(TARGET CREATE_LLVM_NATIVE PROPERTY FOLDER Misc)
         endif()
 #if _WIN32
-        if (MSVC)
-            # We can't use LLVM_OPTIMIZED_TABLEGEN on Windows, and the 32-bit llvm-tblgen can easily
-            # to run out of memory. Tell the linker to allow addresses larger than 2GB.
+        if(MSVC)
+            # The 32-bit llvm-tblgen can easily run out of memory. Tell the linker to allow addresses larger than 2GB.
             set_property(TARGET llvm-tblgen PROPERTY LINK_FLAGS "/LARGEADDRESSAWARE")
         endif()
 #endif

cmake/LlvmMainRevision.cmake

@@ -0,0 +1,44 @@
+##
+ #######################################################################################################################
+ #
+ #  Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights Reserved.
+ #
+ #  Permission is hereby granted, free of charge, to any person obtaining a copy
+ #  of this software and associated documentation files (the "Software"), to
+ #  deal in the Software without restriction, including without limitation the
+ #  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ #  sell copies of the Software, and to permit persons to whom the Software is
+ #  furnished to do so, subject to the following conditions:
+ #
+ #  The above copyright notice and this permission notice shall be included in all
+ #  copies or substantial portions of the Software.
+ #
+ #  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ #  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ #  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ #  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ #  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ #  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ #  IN THE SOFTWARE.
+ #
+ #######################################################################################################################
+
+# Include this file to set LLVM_MAIN_REVISION, for when it is needed at cmake level rather than C++ level.
+
+if (NOT LLVM_MAIN_REVISION)
+  # A sneaky way to get the LLVM source directory, assuming we are included from a LLVM external
+  # project such as LGC or LLPCFE.
+  get_filename_component(LLVM_SOURCE_DIR "${CPACK_RESOURCE_FILE_LICENSE}" DIRECTORY)
+  if (NOT LLVM_SOURCE_DIR)
+      message(FATAL_ERROR "LLVM_SOURCE_DIR not found")
+  endif()
+
+  # Scrape LLVM_MAIN_REVISION out of llvm-config.h.cmake. If not found, set to a high number.
+  set(LLVM_CONFIG_H_NAME "${LLVM_SOURCE_DIR}/include/llvm/Config/llvm-config.h.cmake")
+  file(READ "${LLVM_CONFIG_H_NAME}" LLVM_CONFIG_H_CONTENTS)
+  string(REGEX REPLACE "^.* LLVM_MAIN_REVISION ([0-9]+).*$" "\\1" LLVM_MAIN_REVISION "${LLVM_CONFIG_H_CONTENTS}")
+  if ("${LLVM_MAIN_REVISION}" STREQUAL "${LLVM_CONFIG_H_CONTENTS}")
+      set(LLVM_MAIN_REVISION 999999999)
+  endif()
+endif()
+

compilerutils/include/compilerutils/CompilerUtils.h

@@ -92,7 +92,20 @@ struct CrossModuleInlinerResult {
 // One CrossModuleInliner instance must only be used for a single target module, otherwise things can go wrong.
 class CrossModuleInliner {
 public:
-  CrossModuleInliner() = default;
+  // Callback passed to getGlobalInModule, that tries to find an existing GlobalValue in the target module or copies it
+  // to the target module.
+  using GetGlobalInModuleTy = std::function<llvm::GlobalValue &(CrossModuleInliner &inliner,
+                                                                llvm::GlobalValue &sourceGV, llvm::Module &targetGv)>;
+
+  CrossModuleInliner(GetGlobalInModuleTy getGlobalInModuleCallback = defaultGetGlobalInModuleFunc);
+
+  // Do not allow copy but allow moving
+  CrossModuleInliner(const CrossModuleInliner &) = delete;
+  CrossModuleInliner(CrossModuleInliner &&);
+  CrossModuleInliner &operator=(const CrossModuleInliner &) = delete;
+  CrossModuleInliner &operator=(CrossModuleInliner &&);
+
+  ~CrossModuleInliner() noexcept;
 
   // Inline a call to a function even if the called function is in a different module.
   // If the result of that function call should be used, a use must exist before calling this function.
@@ -118,19 +131,21 @@ class CrossModuleInliner {
   // target module.
   llvm::GlobalValue *findCopiedGlobal(llvm::GlobalValue &sourceGv, llvm::Module &targetModule);
 
+  // Default implementation that finds global values using getCrossModuleName.
+  static llvm::GlobalValue &defaultGetGlobalInModuleFunc(CrossModuleInliner &inliner, llvm::GlobalValue &sourceGv,
+                                                         llvm::Module &targetModule);
+
   static std::string getCrossModuleName(llvm::GlobalValue &gv);
 
 private:
   // Checks that we haven't processed a different target module earlier.
-  void checkTargetModule(llvm::Module &targetModule) {
-    if (lastUsedTargetModule == nullptr)
-      lastUsedTargetModule = &targetModule;
-    else
-      assert(lastUsedTargetModule == &targetModule);
-  }
+  void checkTargetModule(llvm::Module &targetModule);
+
+  struct Impl;
+  class CrossModuleValueMaterializer;
 
-  llvm::SmallDenseMap<llvm::GlobalValue *, llvm::GlobalValue *> mappedGlobals;
-  llvm::Module *lastUsedTargetModule = nullptr; // used to check that we don't use different target modules
+  // Split into Impl class, so we don’t need to include everything in this header.
+  std::unique_ptr<Impl> impl;
 };
 
 // Essentially RAUW for pointers for the case that these use different address
@@ -140,6 +155,13 @@ class CrossModuleInliner {
 // The caller has to handle the erasure afterwards.
 void replaceAllPointerUses(llvm::IRBuilder<> *builder, llvm::Value *oldPointerValue, llvm::Value *newPointerValue,
                            llvm::SmallVectorImpl<llvm::Instruction *> &toBeRemoved);
+
+// Create a GEP if idx is non-null, otherwise return the pointer.
+llvm::Value *simplifyingCreateConstGEP1_32(llvm::IRBuilder<> &builder, llvm::Type *ty, llvm::Value *ptr, uint32_t idx);
+
+// Create an inbounds GEP if idx is non-null, otherwise return the pointer.
+llvm::Value *simplifyingCreateConstInBoundsGEP1_32(llvm::IRBuilder<> &builder, llvm::Type *ty, llvm::Value *ptr,
+                                                   uint32_t idx);
 } // namespace CompilerUtils
 
 namespace llvm {

compilerutils/include/compilerutils/DxilUtils.h

@@ -0,0 +1,64 @@
+/*
+ ***********************************************************************************************************************
+ *
+ *  Copyright (c) 2024 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ *  Permission is hereby granted, free of charge, to any person obtaining a copy
+ *  of this software and associated documentation files (the "Software"), to
+ *  deal in the Software without restriction, including without limitation the
+ *  rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ *  sell copies of the Software, and to permit persons to whom the Software is
+ *  furnished to do so, subject to the following conditions:
+ *
+ *  The above copyright notice and this permission notice shall be included in all
+ *  copies or substantial portions of the Software.
+ *
+ *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ *  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ *  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ *  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ *  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ *  FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ *  IN THE SOFTWARE.
+ *
+ **********************************************************************************************************************/
+
+//===- DxilUtils.h -  --------------------------------------------------------------------------------------------===//
+//
+// Shared DXIl-related helpers.
+//
+//===--------------------------------------------------------------------------------------------------------------===//
+
+#pragma once
+
+#include "llvm/ADT/StringRef.h"
+
+namespace CompilerUtils::dxil {
+
+// Try to demangle function names in the DXIL format:
+// ...\01?FuncName@@...
+// @param funcName : Name of the callee
+// @returns: the original string if the name was not demangleable or the demangled function name.
+inline llvm::StringRef tryDemangleFunctionName(llvm::StringRef inputName) {
+  assert(!inputName.empty());
+
+  constexpr static llvm::StringRef manglingPrefix = "\01?";
+
+  // Expect both characters to be there, and `\01?` to occur before `@@`
+  size_t start = inputName.find(manglingPrefix);
+  if (start == llvm::StringRef::npos)
+    return inputName;
+
+  // The case start >= end is implicitly checked by the second call to `find`.
+  const size_t end = inputName.find("@@", start);
+  if (end == llvm::StringRef::npos)
+    return inputName;
+
+  start += manglingPrefix.size();
+
+  // Extract unmangled name: Return everything after the first occurrence of `\01?` and before the first occurrence of
+  // `@@` after `?`.
+  return inputName.substr(start, end - start);
+}
+
+} // namespace CompilerUtils::dxil

compilerutils/include/compilerutils/ValueOriginTracking.h

@@ -106,9 +106,12 @@ namespace ValueTracking {
 struct SliceStatus {
   // As the actual enum is contained within the struct, its values don't leak into the containing namespace,
   // and it's not possible to implicitly cast a SliceStatus to an int, so it's as good as an enum class.
+  // The UndefOrPoison case always originates from a `poison` or `undef` value.
+  // We must be careful with freeze instructions operating on such values, see FreezeHandlingMode.
   enum StatusEnum : uint32_t { Constant = 0x1, Dynamic = 0x2, UndefOrPoison = 0x4 };
   StatusEnum S = {};
 
+  // Intentionally allow implicit conversion:
   SliceStatus(StatusEnum S) : S{S} {}
 
   static SliceStatus makeEmpty() { return static_cast<StatusEnum>(0); }
@@ -188,6 +191,45 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const ValueInfo &VI);
 // constant and then always propagated, allowing to replace the argument by the initial constant.
 class ValueOriginTracker {
 public:
+  // Configuration options for ValueOriginTracker.
+  struct Options {
+    unsigned BytesPerSlice = 4;
+    unsigned MaxBytesPerValue = 512;
+
+    // Freeze instructions are problematic for value origin tracking.
+    //
+    // While `freeze poison` are intended to help optimization by allowing it to pick any value, we cannot just
+    // treat `freeze poison` as UndefOrPoison, because an optimization relying on that would need to ensure
+    // other users of the optimized `freeze poison` observe the same value picked by optimization, and value origin
+    // tracking does not allow to query which `freeze poison` instructions a particular slice originates from.
+    // Instead, the only safe way to treat `freeze poison` is dynamic.
+    //
+    // In some cases, e.g. when not optimizing based on the analysis result, and instead just using it for sanity
+    // checking in testing, treating `freeze poison` as UndefOrPoison however is the intended result, and if
+    // value origin tracking implicitly considered all `freeze poison` as dynamic, then client code would need to
+    // propagate the intended UndefOrPoison semantics manually.
+    //
+    // The FreezeHandlingMode enum allows to avoid that, allowing the client to specify how `freeze poison` and
+    // `freeze undef` should be handled.
+    //
+    // If we want to optimize based on `freeze poison`, one option would be eliminating all freeze instructions by some
+    // constant (e.g. `zeroinitializer`) before running the analysis, as some LLVM transforms like instcombine do.
+    // This ensures that not only the analysis sees a common constant value for `freeze poison`, but also ensures other
+    // uses of `freeze poison` observe the same value.
+    //
+    // As a less conservative potential future improvement, we could instead explicitly keep track of FrozenPoison
+    // slices in value origin tracking, and when merging FrozenPoison with constants, recording which `freeze poison`
+    // values need to be replaced by which constants to allow that.
+    enum class FreezeHandlingMode {
+      // Treat slices in freeze instructions that are UndefOrPoison in the freeze operand as dynamic.
+      Dynamic = 0,
+      // Always forward value infos of freeze operands for freeze instructions.
+      // In particular, `freeze poison` is always reported as UndefOrPoison.
+      Forward
+    };
+    FreezeHandlingMode FreezeMode = FreezeHandlingMode::Dynamic;
+  };
+
   using ValueInfo = ValueTracking::ValueInfo;
   // In some cases, client code has additional information on where values originate from, or
   // where they should be assumed to originate from just for the purpose of the analysis.
@@ -225,10 +267,9 @@ class ValueOriginTracker {
   // Also, only a single status on assumptions is allowed.
   using ValueOriginAssumptions = llvm::DenseMap<llvm::Instruction *, ValueInfo>;
 
-  ValueOriginTracker(const llvm::DataLayout &DL, unsigned BytesPerSlice = 4, unsigned MaxBytesPerValue = 512,
+  ValueOriginTracker(const llvm::DataLayout &DL, Options Opts,
                      ValueOriginAssumptions OriginAssumptions = ValueOriginAssumptions{})
-      : DL{DL}, BytesPerSlice{BytesPerSlice}, MaxBytesPerValue{MaxBytesPerValue},
-        OriginAssumptions(std::move(OriginAssumptions)) {}
+      : DL{DL}, Opts{Opts}, OriginAssumptions(std::move(OriginAssumptions)) {}
 
   // Computes a value info for the given value.
   // If the value has been seen before, returns a cache hit from the ValueInfos map.
@@ -247,8 +288,7 @@ class ValueOriginTracker {
 private:
   struct ValueInfoBuilder;
   const llvm::DataLayout &DL;
-  unsigned BytesPerSlice = 0;
-  unsigned MaxBytesPerValue = 0;
+  Options Opts;
   ValueOriginAssumptions OriginAssumptions;
   llvm::DenseMap<llvm::Value *, ValueInfo> ValueInfos;