diff --git a/.github/Dockerfile b/.github/Dockerfile new file mode 100644 index 000000000..917073430 --- /dev/null +++ b/.github/Dockerfile @@ -0,0 +1,76 @@ +FROM ubuntu:22.04 +SHELL ["/bin/bash", "-c"] + +# Set environment variables +ENV DEBIAN_FRONTEND=noninteractive +ENV TTMLIR_TOOLCHAIN_DIR=/opt/ttmlir-toolchain +ENV TTFORGE_TOOLCHAIN_DIR=/opt/ttforge-toolchain +ENV PROJECT_NAME=tt-forge +ARG GITHUB_TOKEN + +# Install dependencies +RUN apt-get update && apt-get install -y \ + software-properties-common \ + build-essential \ + python3-dev \ + python3-venv \ + python3-pip \ + git \ + git-lfs \ + libhwloc-dev \ + pandoc \ + libtbb-dev \ + libcapstone-dev \ + pkg-config \ + linux-tools-generic \ + ninja-build \ + wget \ + cmake \ + ccache \ + doxygen \ + libgtest-dev \ + libgmock-dev \ + graphviz \ + patchelf \ + libyaml-cpp-dev \ + libboost-all-dev + +# Install clang 17 +RUN wget https://apt.llvm.org/llvm.sh && \ + chmod u+x llvm.sh && \ + ./llvm.sh 17 && \ + apt install -y libc++-17-dev libc++abi-17-dev && \ + ln -s /usr/bin/clang-17 /usr/bin/clang && \ + ln -s /usr/bin/clang++-17 /usr/bin/clang++ + +# Install python packages +RUN pip install cmake \ + pytest + +# Create a directory for the build and toolchain +ARG BUILD_DIR=/home/build +RUN mkdir -p $BUILD_DIR && \ + mkdir -p $TTMLIR_TOOLCHAIN_DIR && \ + mkdir -p $TTFORGE_TOOLCHAIN_DIR + +# Clone the project and update submodules +# Pass in PAT to clone private repositories +RUN git clone https://$GITHUB_TOKEN@github.com/tenstorrent/tt-forge.git $BUILD_DIR/$PROJECT_NAME && \ + cd $BUILD_DIR/$PROJECT_NAME && \ + git submodule update --init --recursive -f + +# Build the toolchain +WORKDIR $BUILD_DIR/$PROJECT_NAME +RUN source env/activate && \ + cmake -B env/build env && \ + cmake --build env/build + +# Build project to test the container +RUN source env/activate && \ + cmake -G Ninja -B build . && \ + cmake --build build + +# Clean up the build directory +RUN cd .. && \ + rm -rf $BUILD_DIR/$PROJECT_NAME && \ + unset GITHUB_TOKEN \ No newline at end of file diff --git a/.github/actions/install-deps/action.yml b/.github/actions/install-deps/action.yml deleted file mode 100644 index 1f13767d9..000000000 --- a/.github/actions/install-deps/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: "Install dependencies" -description: "Installs dependencies on GitHub Actions runners" - -inputs: - os: - description: 'Runner OS' - required: true - -runs: - using: "composite" - steps: - - name: Verify ubuntu only - shell: bash - run: | - if ! echo ${{ inputs.os }} | grep -q "ubuntu"; then - echo "${{ inputs.os }} does not seem to be ubuntu" - fi - - name: Assert requested os exists in dependencies - shell: bash - run: | - if ! jq -e ".\"${{ inputs.os }}\" != null" $GITHUB_ACTION_PATH/dependencies.json; then - echo "${{ inputs.os }} does not exist as a supported os in $GITHUB_ACTION_PATH/dependencies.json" - fi - - name: Retrieve and install pkg deps based on OS - id: retrieve-pkg-deps - shell: bash - run: | - DEPENDENCIES=$(jq -r --arg os "${{ inputs.os }}" '.[$os] | .[]' $GITHUB_ACTION_PATH/dependencies.json) - echo $DEPENDENCIES - sudo apt update - sudo apt install $DEPENDENCIES diff --git a/.github/actions/install-deps/dependencies.json b/.github/actions/install-deps/dependencies.json deleted file mode 100644 index 2faab1678..000000000 --- a/.github/actions/install-deps/dependencies.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "ubuntu-22.04": [ - "software-properties-common", - "build-essential", - "python3.10-venv", - "libyaml-cpp-dev", - "libboost-all-dev", - "libsndfile1", - "libhwloc-dev", - "libzmq3-dev" - ] -} diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml deleted file mode 100644 index 5e52e7e8f..000000000 --- a/.github/workflows/build-artifacts.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Build artifacts - -on: - workflow_dispatch: - workflow_call: - -env: - PYTHON_VERSION: "python3.10" - -jobs: - build-artifacts: - strategy: - matrix: - arch: ["grayskull"] - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/install-deps - with: - os: ubuntu-22.04 - - name: Update submodule - run: git submodule update --init --recursive - - name: Build for ${{ matrix.arch }} - run: source env_for_silicon.sh \ No newline at end of file diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml new file mode 100644 index 000000000..0f60a7306 --- /dev/null +++ b/.github/workflows/build-image.yml @@ -0,0 +1,44 @@ +name: Build and Publish Docker Image + +on: + workflow_dispatch: + workflow_call: + +jobs: + build: + runs-on: self-hosted # Github runners dont have enough storage space to build this image + + steps: + + - run: | + echo Pwd: $(pwd) + echo User: $(whoami) + ls -la /home/ubuntu/actions-runner/_work/tt-forge/tt-forge + shell: bash + + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: | + .github/Dockerfile + sparse-checkout-cone-mode: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: .github + file: .github/Dockerfile + push: true + tags: | + ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:${{ github.sha }} + ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..365fbe546 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,75 @@ +name: Build in Docker + +on: + workflow_dispatch: + workflow_call: + +jobs: + + build-and-test: + + strategy: + fail-fast: false + matrix: + build: + - runs-on: self-hosted + + runs-on: ${{ matrix.build.runs-on }} + + container: + image: ghcr.io/${{ github.repository }}/tt-forge-ubuntu-22-04:latest + options: --user root --device /dev/tenstorrent/0 + volumes: + - /dev/hugepages:/dev/hugepages + - /dev/hugepages-1G:/dev/hugepages-1G + - /etc/udev/rules.d:/etc/udev/rules.d + - /lib/modules:/lib/modules + - /opt/tt_metal_infra/provisioning/provisioning_env:/opt/tt_metal_infra/provisioning/provisioning_env + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # Fetch all history and tags + + - name: Set reusable strings + id: strings + shell: bash + run: | + echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT" + echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT" + + - name: Git safe dir + run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }} + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + create-symlink: true + key: ${{ matrix.build.runs-on }}-runtime-${{ matrix.build.enable_runtime }}-${{ env.SDK_VERSION }} + + - name: Build + shell: bash + run: | + source env/activate + cmake -G Ninja -B build . + cmake --build build + + - name: Run Test + shell: bash + run: | + source env/activate + pytest + continue-on-error: true + + - name: Upload Test Report + uses: actions/upload-artifact@v4 + with: + name: test-reports-${{ matrix.build.runs-on }} + path: reports/report.xml + + - name: Show Test Report + uses: mikepenz/action-junit-report@v4 + if: success() || failure() + with: + report_paths: reports/report.xml + check_name: TT-Forge Tests diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml new file mode 100644 index 000000000..e179fb633 --- /dev/null +++ b/.github/workflows/on-pr.yml @@ -0,0 +1,11 @@ +name: On PR + +on: + workflow_dispatch: + pull_request: + branches: [ "main" ] + +jobs: + docker-build: + uses: ./.github/workflows/docker-build.yml + secrets: inherit diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml new file mode 100644 index 000000000..1bb669f77 --- /dev/null +++ b/.github/workflows/on-push.yml @@ -0,0 +1,11 @@ +name: On push + +on: + workflow_dispatch: + push: + branches: [ "main" ] + +jobs: + docker-build: + uses: ./.github/workflows/docker-build.yml + secrets: inherit diff --git a/.github/workflows/post-commit-workflow.yml b/.github/workflows/post-commit-workflow.yml deleted file mode 100644 index ceb7d58da..000000000 --- a/.github/workflows/post-commit-workflow.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Post commit workflow - -on: - workflow_dispatch: - workflow_call: - push: - branches: - - main - -jobs: - build-artifacts: - uses: ./.github/workflows/build-artifacts.yml - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/pull-request-workflow.yml b/.github/workflows/pull-request-workflow.yml deleted file mode 100644 index c5fbe7958..000000000 --- a/.github/workflows/pull-request-workflow.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Pull request workflow - -on: - workflow_dispatch: - workflow_call: - pull_request: - branches: - - main - -jobs: - build-artifacts: - uses: ./.github/workflows/build-artifacts.yml - secrets: inherit \ No newline at end of file diff --git a/.gitignore b/.gitignore index 4dff4f040..82131f71a 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,6 @@ build_deps/ # ClangD compile_commands.json \n\n# Exclude LFS files to keep the public repo small + +# Test reports +reports/report.xml diff --git a/.gitlab/issue_templates/API.md b/.gitlab/issue_templates/API.md deleted file mode 100644 index 6c5dcb482..000000000 --- a/.gitlab/issue_templates/API.md +++ /dev/null @@ -1,17 +0,0 @@ -## API description - -*(Brief usage guide and motivations)* - -## Adoption Impact - -*(Breakdown the impact for adopting the new API and impact on existing FE codebase)* - -## Deprecation -*(Delete this section for new API, or changes with no API interface impact)* - -Procedure: -- FE integrates BE, old interface is still used -- FE implements new API after BE uplift merged to main -- Re-assign this issue on BE to remove the API - -*(List the APIs being replaced and implementation suggetsions)* diff --git a/.gitlab/issue_templates/Default.md b/.gitlab/issue_templates/Default.md deleted file mode 100644 index d4405aec4..000000000 --- a/.gitlab/issue_templates/Default.md +++ /dev/null @@ -1,21 +0,0 @@ -## Description - -*(Summarize the bug encountered concisely)* - -## Reproduce - -*(How one can reproduce the issue - this is very important)* - -```bash -repro command -``` - -## Observed behavior - -*(What actually happens)* - -## Expected behavior - -*(What you should see instead)* - -/label ~state::Screen diff --git a/.gitlab/issue_templates/Feature.md b/.gitlab/issue_templates/Feature.md deleted file mode 100644 index 6e2fcf2d7..000000000 --- a/.gitlab/issue_templates/Feature.md +++ /dev/null @@ -1,15 +0,0 @@ -## Overview - -*(Summarize the feature)* - -## Motivation - -*(Outline what this feature intends to achieve)* - -## Implementation / Code Impact - -*(Outline how the feature might be implemented + impacted code components)* - -## Test Plan - -*(List of test cases to implement / models that cover this feature)* diff --git a/env/CMakeLists.txt b/env/CMakeLists.txt index ebeb8fce2..b93219cd5 100644 --- a/env/CMakeLists.txt +++ b/env/CMakeLists.txt @@ -36,3 +36,12 @@ add_custom_target(python-venv CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR} TTFORGE_VENV_DIR=${TTFORGE_VENV_DIR} bash ${CMAKE_CURRENT_SOURCE_DIR}/create_venv.sh) + +add_custom_target(build_tt_mlir_env ALL + COMMAND ${CMAKE_COMMAND} -E env + TTMLIR_TOOLCHAIN_DIR=${TTMLIR_TOOLCHAIN_DIR} # Export TTMLIR_TOOLCHAIN_DIR to use + TTMLIR_VENV_DIR=${TTMLIR_VENV_DIR} # Export TTMLIR_VENV_DIR to use + bash ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/build_mlir_env.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/tt-mlir + USES_TERMINAL +) \ No newline at end of file diff --git a/pybuda/test/mlir/test_ops.py b/pybuda/test/mlir/test_ops.py index bcb145fda..d3c2ddc00 100644 --- a/pybuda/test/mlir/test_ops.py +++ b/pybuda/test/mlir/test_ops.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import pytest import torch from torch import nn @@ -89,7 +90,7 @@ def forward(self, a): co_out = [co.to("cpu") for co in co_out] assert [torch.allclose(fo, co) for fo, co in zip(fw_out, co_out)] - +@pytest.mark.skip(reason="This is not ready yet") def test_linear(): class Linear(nn.Module): def __init__(self): diff --git a/pytest.ini b/pytest.ini index e8d0284e0..a6924e5b5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,13 @@ # TF & some other libraries report a bunch of deprecation warnings [pytest] + +# Ignore specific tests +addopts = -svv --junit-xml=reports/report.xml + +# Where pytest should look for tests +testpaths = + pybuda/test/mlir/test_ops.py + pybuda/test/test_api.py + filterwarnings = ignore::DeprecationWarning diff --git a/third_party/build_mlir.sh b/third_party/build_mlir.sh index 8078e095f..2f4c9a648 100644 --- a/third_party/build_mlir.sh +++ b/third_party/build_mlir.sh @@ -5,9 +5,6 @@ source env/activate -cmake -B env/build env -cmake --build env/build - build_type=${BUILD_TYPE:-Release} c_compiler=${C_COMPILER:-clang} cxx_compiler=${CXX_COMPILER:-clang++} diff --git a/third_party/build_mlir_env.sh b/third_party/build_mlir_env.sh new file mode 100644 index 000000000..0adc0d5ed --- /dev/null +++ b/third_party/build_mlir_env.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-FileCopyrightText: (c) 2024 Tenstorrent AI ULC +# +# SPDX-License-Identifier: Apache-2.0 + +source env/activate + +cmake -B env/build env +cmake --build env/build