Somewhat hacky way to build toolchain

tenstorrent · Sep 17, 2024 · a26338f · a26338f
1 parent cb05921
commit a26338f
Show file tree

Hide file tree

Showing 2 changed files with 70 additions and 48 deletions.
diff --git a/.github/Dockerfile.ci b/.github/Dockerfile.ci
@@ -16,13 +16,15 @@ RUN mkdir -p $BUILD_DIR && \
     mkdir -p $TTMLIR_TOOLCHAIN_DIR
 
 # Clone tt-mlir
-RUN git clone https://github.com/tenstorrent/tt-mlir.git $BUILD_DIR/$PROJECT_NAME && \
+RUN git clone https://github.com/tenstorrent/$PROJECT_NAME.git $BUILD_DIR/$PROJECT_NAME && \
     cd $BUILD_DIR/$PROJECT_NAME && \
     git checkout $GIT_SHA
 
 # Build the toolchain
 WORKDIR $BUILD_DIR/$PROJECT_NAME
-RUN source env/activate && \
+RUN cmake -B toolchain -DTOOLCHAIN=ON third_party/ && \
+    cd third_party/tt-mlir/src/tt-mlir/ && \
+    source env/activate && \
     cmake -B env/build env && \
     cmake --build env/build
 

diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
@@ -1,50 +1,70 @@
-include(ExternalProject)
-add_subdirectory(pjrt_c_api)
+set(TT_MLIR_VERSION "f955a08e8836e41a3333ca20a3bbe193ce91b2a0")
+set(LOGURU_VERSION "4adaa185883e3c04da25913579c451d3c32cfac1")
 
-set(TTMLIR_LIB_DIR ${PROJECT_SOURCE_DIR}/third_party/tt-mlir/src/tt-mlir-build/lib/SharedLib)
-ExternalProject_Add(
-    tt-mlir
-    PREFIX ${TTPJRT_SOURCE_DIR}/third_party/tt-mlir
-    CMAKE_GENERATOR Ninja
-    CMAKE_ARGS
-      -DCMAKE_BUILD_TYPE=Release
-      -DCMAKE_C_COMPILER=clang-17
-      -DCMAKE_CXX_COMPILER=clang++-17
-      -DTT_RUNTIME_ENABLE_TTNN=ON
-      -DTTMLIR_ENABLE_STABLEHLO=ON
-      -DTTMLIR_ENABLE_RUNTIME=ON
-    GIT_REPOSITORY [email protected]:tenstorrent/tt-mlir.git
-    GIT_TAG f955a08e8836e41a3333ca20a3bbe193ce91b2a0
-    GIT_PROGRESS ON
-    INSTALL_COMMAND ""
-)
+if (TOOLCHAIN STREQUAL "ON")
+    cmake_minimum_required(VERSION 3.20)
+    project(ttmlir-toolchain LANGUAGES CXX C)
+    execute_process(
+        COMMAND git clone --recursive [email protected]:tenstorrent/tt-mlir.git ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir
+        RESULT_VARIABLE result1
+        OUTPUT_VARIABLE output1
+        ERROR_VARIABLE error_output1
+    )
+    if (NOT ${result1} EQUAL 0)
+        message(FATAL_ERROR "Failed to clone tt-mlir: ${error_output1}")
+    endif()
+    execute_process(
+        COMMAND /bin/bash -c "cd ${PROJECT_SOURCE_DIR}/tt-mlir/src/tt-mlir && git checkout ${TT_MLIR_VERSION}"
+    )
+else()
+    include(ExternalProject)
+    add_subdirectory(pjrt_c_api)
+    set(TTMLIR_LIB_DIR ${PROJECT_SOURCE_DIR}/third_party/tt-mlir/src/tt-mlir-build/lib/SharedLib)
+    ExternalProject_Add(
+        tt-mlir
+        PREFIX ${TTPJRT_SOURCE_DIR}/third_party/tt-mlir
+        CMAKE_GENERATOR Ninja
+        CMAKE_ARGS
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_C_COMPILER=clang-17
+          -DCMAKE_CXX_COMPILER=clang++-17
+          -DTT_RUNTIME_ENABLE_TTNN=ON
+          -DTTMLIR_ENABLE_STABLEHLO=ON
+          -DTTMLIR_ENABLE_RUNTIME=ON
+        GIT_REPOSITORY [email protected]:tenstorrent/tt-mlir.git
+        GIT_TAG ${TT_MLIR_VERSION}
+        GIT_PROGRESS ON
+        INSTALL_COMMAND ""
+    )
 
-set_target_properties(tt-mlir PROPERTIES EXCLUDE_FROM_ALL TRUE)
+    set_target_properties(tt-mlir PROPERTIES EXCLUDE_FROM_ALL TRUE)
 
-# Link to all TTMLIR libraries in ${TTMLIR_INSTALL_DIR}/install/lib
-set(TTMLIR_LIB_DIR ${TTMLIR_LIB_DIR} PARENT_SCOPE)
-message(STATUS "Linking to TTMLIR libraries in ${TTMLIR_LIB_DIR}")
-file(GLOB TTMLIR_LIBRARIES "${TTMLIR_LIB_DIR}/*.so")
-foreach(TTMLIR_LIBRARY ${TTMLIR_LIBRARIES})
-    get_filename_component(lib_name ${TTMLIR_LIBRARY} NAME_WE)
-    string(REPLACE "lib" "" lib_name ${lib_name}) # Remove the "lib" prefix if it exists
-    add_library(${lib_name} SHARED IMPORTED GLOBAL)
-    set_target_properties(${lib_name} PROPERTIES EXCLUDE_FROM_ALL TRUE IMPORTED_LOCATION ${TTMLIR_LIBRARY})
-    add_dependencies(${lib_name} tt-mlir)
-endforeach()
+    # Link to all TTMLIR libraries in ${TTMLIR_INSTALL_DIR}/install/lib
+    set(TTMLIR_LIB_DIR ${TTMLIR_LIB_DIR} PARENT_SCOPE)
+    message(STATUS "Linking to TTMLIR libraries in ${TTMLIR_LIB_DIR}")
+    file(GLOB TTMLIR_LIBRARIES "${TTMLIR_LIB_DIR}/*.so")
+    foreach(TTMLIR_LIBRARY ${TTMLIR_LIBRARIES})
+        get_filename_component(lib_name ${TTMLIR_LIBRARY} NAME_WE)
+        string(REPLACE "lib" "" lib_name ${lib_name}) # Remove the "lib" prefix if it exists
+        add_library(${lib_name} SHARED IMPORTED GLOBAL)
+        set_target_properties(${lib_name} PROPERTIES EXCLUDE_FROM_ALL TRUE IMPORTED_LOCATION ${TTMLIR_LIBRARY})
+        add_dependencies(${lib_name} tt-mlir)
+    endforeach()
 
-set(FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-undef -Wno-unsafe-buffer-usage -Wno-disabled-macro-expansion")
-ExternalProject_Add(
-    loguru
-    PREFIX ${TTPJRT_SOURCE_DIR}/third_party/loguru
-    CMAKE_GENERATOR Ninja
-    CMAKE_ARGS
-      -DCMAKE_BUILD_TYPE=Release
-      -DCMAKE_C_COMPILER=clang-17
-      -DCMAKE_CXX_COMPILER=clang++-17
-      -DCMAKE_CXX_FLAGS=${FLAGS}
-      -DCMAKE_INSTALL_PREFIX=${TTPJRT_SOURCE_DIR}/third_party/loguru/src/loguru-install
-    GIT_REPOSITORY [email protected]:emilk/loguru.git
-    GIT_TAG 4adaa185883e3c04da25913579c451d3c32cfac1
-    GIT_PROGRESS ON
-)
+    set(FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-undef -Wno-unsafe-buffer-usage -Wno-disabled-macro-expansion")
+    ExternalProject_Add(
+        loguru
+        PREFIX ${TTPJRT_SOURCE_DIR}/third_party/loguru
+        CMAKE_GENERATOR Ninja
+        CMAKE_ARGS
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_C_COMPILER=clang-17
+          -DCMAKE_CXX_COMPILER=clang++-17
+          -DCMAKE_CXX_FLAGS=${FLAGS}
+          -DCMAKE_INSTALL_PREFIX=${TTPJRT_SOURCE_DIR}/third_party/loguru/src/loguru-install
+        GIT_REPOSITORY [email protected]:emilk/loguru.git
+        GIT_TAG ${LOGURU_VERSION}
+        GIT_PROGRESS ON
+    )
+
+endif()