Docs - Build & Deploy

update requirements (#904) #316

Triggered via push December 17, 2024 10:17

mstojkovicTT

pushed a194ca3

main

Status Success

Total duration 6m 4s

Artifacts –

docs.yml

on: push

Matrix: build

deploy

Annotations

101 errors and 2 warnings

test_training.test_mnist_training: forge/test/mlir/mnist/training/test_training.py#L68

RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/program/program.cpp:819: tt::exception info: Statically allocated circular buffers on core range [(x=0,y=0) - (x=3,y=0)] grow to 1671968 B which is beyond max L1 size of 1499136 B backtrace: --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/libtt_metal.so(+0x14d79b) [0x7fe81c1a979b] --- tt::tt_metal::detail::Program_::validate_circular_buffer_region(tt::tt_metal::v0::Device const*) --- tt::tt_metal::v0::CommandQueue::run_command_impl(tt::tt_metal::CommandInterface const&) --- tt::tt_metal::v0::EnqueueProgram(tt::tt_metal::v0::CommandQueue&, tt::tt_metal::v0::Program&, bool) --- void ttnn::device_operation::detail::launch_on_worker_thread<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >, tt::tt_metal::v0::Device*>(unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >&, tt::tt_metal::v0::Device*&) --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::launch_on_single_device<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1de51c9) [0x7fe80a8351c9] --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::invoke<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1de1aef) [0x7fe80a831aef] --- std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > tt::tt_metal::operation::run<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >(tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >&&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > const&, std::vector<std::optional<tt::tt_metal::Tensor const>, std::allocator<std::optional<tt::tt_metal::Tensor const> > > const&, std::vector<std::optional<tt::tt_metal::Tensor>, std::allocator<std::optional<tt::tt_metal::Tensor> > > const&, unsigned char) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x140dfd3) [0x7fe809e5dfd3] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x140ea95) [0x7fe809e5ea95] --- ttnn::operations::matmul::matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char) --- ttnn::operations::matmul::bound_matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char const&) --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(_ZN4ttnn10operations6matmul15MatmulOperation6invokeERKN2tt8tt_metal6TensorES7_bbRKSt8optionalIKNS4_12MemoryConfigEES8_IKNS4_8DataTypeEERKS8_IKSt7variantIJNS1_28MatmulMultiCoreProgramConfigENS1_45MatmulMultiCoreNonOptimizedReuseProgramConfigENS1_33MatmulMultiCoreReuseProgramConfigENS1_42MatmulMultiCoreReuseMultiCastProgramConfigENS1_44MatmulMultiCoreReuseMultiCast1DProgramConfigENS1_53MatmulMultiCoreReuseMultiCastDRAMShardedProgramConfigEEEERKS8_IKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEES8_IKSH_IJNS_28GrayskullComputeKernelConfigENS_27WormholeComputeKernelConfigEEEES8_IKNS_5types8CoreGridEERKS8_IKNS4_4TileEE+0x2c5) [0x7fe809df37c5] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x77532b) [0x7fe80bd5432b] --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7745fd) [0x7fe80bd535fd] --- tt::runtime::ttnn::operations::matmul::run(tt::target::ttnn::MatmulOp const*, tt::runtime::ttnn::ProgramContext&) --- tt::runtime::ttnn::legacy::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary&, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&) --- tt::runtime::ttnn::legacy::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&) --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&) --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&) --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x311950) [0x7fe81c8d5950] --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31189e) [0x7fe81c8d589e] --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf8d5) [0x7fe81c6a38d5] --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x5585400d3282] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5585400c9b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x5585400c2f4d] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x5585400c8d14] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5585400ddd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x5585401e6054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5585400c9b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x5585400c2f4d] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x5585400e0be1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5585400c8dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5585400ddd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x5585401e6054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5585400c9b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x5585400c39ea] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x5585400bdae8] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x5585400e0be1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5585400c8dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5585400ddd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x5585401e6054] --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x5585400e181b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x5585400bd9a2] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x5585400e0be1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x5585400bd9a2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x5585400e0be1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x5585400c8dbd] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x5585400ddd4c] --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x5585401e6054] --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x5585400c9b4b] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x5585400c39ea] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x5585400bff59] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x5585400becf2] --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x5585400d3aec] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a] --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x5585400e0be1] --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x5585400c2b7a]

test_all.test_plan[no_device-concatenate-FROM_ANOTHER_OP-{'dim': 2}-(1, 3, 4)-2-None-None]: forge/test/operators/pytorch/test_all.py#L1