RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/program/program.cpp:819: tt::exception info: Statically allocated circular buffers on core range [(x=0,y=0) - (x=3,y=0)] grow to 1671968 B which is beyond max L1 size of 1499136 B backtrace: --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/libtt_metal.so(+0x14d99b) [0x7fd808d0699b] --- tt::tt_metal::detail::Program_::validate_circular_buffer_region(tt::tt_metal::v0::Device const*) --- tt::tt_metal::v0::CommandQueue::run_command_impl(tt::tt_metal::CommandInterface const&) --- tt::tt_metal::v0::EnqueueProgram(tt::tt_metal::v0::CommandQueue&, tt::tt_metal::v0::Program&, bool) --- void ttnn::device_operation::detail::launch_on_worker_thread<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >, tt::tt_metal::v0::Device*>(unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >&, tt::tt_metal::v0::Device*&) --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::launch_on_single_device<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1e15769) [0x7fd7f77c7769] --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::invoke<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1e1208f) [0x7fd7f77c408f] --- std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > tt::tt_metal::operation::run<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >(tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >&&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > const&, std::vector<std::optional<tt::tt_metal::Tensor const>, std::allocator<std::optional<tt::tt_metal::Tensor const> > > const&, std::vector<std::optional<tt::tt_metal::Tensor>, std::allocator<std::optional<tt::tt_metal::Tensor> > > const&, unsigned char) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1429f03) [0x7fd7f6ddbf03] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x142a9c5) [0x7fd7f6ddc9c5] --- ttnn::operations::matmul::matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char) --- ttnn::operations::matmul::bound_matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(_ZN4ttnn10operations6matmul15MatmulOperation6invokeERKN2tt8tt_metal6TensorES7_bbRKSt8optionalIKNS4_12MemoryConfigEES8_IKNS4_8DataTypeEERKS8_IKSt7variantIJNS1_28MatmulMultiCoreProgramConfigENS1_45MatmulMultiCoreNonOptimizedReuseProgramConfigENS1_33MatmulMultiCoreReuseProgramConfigENS1_42MatmulMultiCoreReuseMultiCastProgramConfigENS1_44MatmulMultiCoreReuseMultiCast1DProgramConfigENS1_53MatmulMultiCoreReuseMultiCastDRAMShardedProgramConfigEEEERKS8_IKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEES8_IKSH_IJNS_28GrayskullComputeKernelConfigENS_27WormholeComputeKernelConfigEEEES8_IKNS_5types8CoreGridEERKS8_IKNS4_4TileEE+0x2c5) [0x7fd7f6d732c5] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7784db) [0x7fd809e704db] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7777ad) [0x7fd809e6f7ad] --- tt::runtime::ttnn::operations::matmul::run(tt::target::ttnn::MatmulOp const*, tt::runtime::ttnn::ProgramContext&) --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&) --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&) --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&) --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&) --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x310600) [0x7fd80a3d5600] --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31054e) [0x7fd80a3d554e] --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf8f5) [0x7fd80a1a48f5] --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x559b0b1c1282] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x559b0b1b7b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x559b0b1b0f4d] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x559b0b1b6d14] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x559b0b1cbd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x559b0b2d4054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x559b0b1b7b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x559b0b1b0f4d] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x559b0b1cebe1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x559b0b1b6dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x559b0b1cbd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x559b0b2d4054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x559b0b1b7b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x559b0b1b19ea] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x559b0b1abae8] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x559b0b1cebe1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x559b0b1b6dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x559b0b1cbd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x559b0b2d4054] --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x559b0b1cf81b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x559b0b1ab9a2] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x559b0b1cebe1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x559b0b1ab9a2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x559b0b1cebe1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x559b0b1b6dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x559b0b1cbd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x559b0b2d4054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x559b0b1b7b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x559b0b1b19ea] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x559b0b1adf59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x559b0b1accf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x559b0b1c1aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x559b0b1cebe1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x559b0b1b0b7a]

ubuntu-latest pipelines will use ubuntu-24.04 soon. For more details, see https://github.com/actions/runner-images/issues/10636