input_shape_and_dim = ((8, 16, 1, 32, 32), 2)

    @pytest.mark.parametrize(
        "input_shape_and_dim",
        [
            ((1, 8, 16, 32, 32), 0),
            ((8, 1, 16, 32, 32), 1),
            ((8, 16, 1, 32, 32), 2),
            ((1, 8, 16, 32, 32), -5),
            ((8, 1, 16, 32, 32), -4),
            ((8, 16, 1, 32, 32), -3),
            ([1, 12, 3200], 0),
            ([1, 1, 2048, 1], [-3, -4]),
            ([1, 64, 1, 1], [-1, -4]),
            ([1, 1, 1, 128], [-2, -4]),
            ([1, 1, 32, 1], [-1, -3]),
            ([1, 1, 1, 64], [-4, -3]),
        ],
    )
    @pytest.mark.push
    def test_squeeze(input_shape_and_dim):
        input_shape, dim = input_shape_and_dim
    
        if input_shape == [1, 12, 3200] or isinstance(dim, list) and len(dim) > 1 and all(d < 0 for d in dim):
            pytest.xfail("TTNN: Tensor layout issues with non tile dim aligned shapes")
    
        class Squeeze(nn.Module):
            def __init__(self):
                super().__init__()
    
            def forward(self, a):
                return torch.squeeze(a, dim)
    
        inputs = [torch.rand(*input_shape)]
    
        framework_model = Squeeze()
        compiled_model = forge.compile(framework_model, sample_inputs=inputs)
    
>       verify(inputs, framework_model, compiled_model)

forge/test/mlir/test_ops.py:1047: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
forge/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f88b3286950>
inputs = (tensor([[[[[0.49626, 0.76822, 0.08848,  ..., 0.37342, 0.30510, 0.93200],
           [0.17591, 0.26983, 0.15068,  ...,...78639,  ..., 0.73204, 0.66506, 0.97919],
           [0.24259, 0.98793, 0.79220,  ..., 0.74142, 0.07995, 0.86774]]]]]),)
inputs_and_parameters = [tensor([[[[[0.49626, 0.76822, 0.08848,  ..., 0.37342, 0.30510, 0.93200],
           [0.17591, 0.26983, 0.15068,  ...,....78639,  ..., 0.73204, 0.66506, 0.97919],
           [0.24259, 0.98793, 0.79220,  ..., 0.74142, 0.07995, 0.86774]]]]])]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*inputs]
        inputs_and_parameters = [
            *inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        if any([not isinstance(t, torch.Tensor) for t in inputs_and_parameters]):
            logger.info("Converting inputs and parameters to PyTorch tensors...")
            inputs_and_parameters = to_pt_tensors(inputs_and_parameters)
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch module.
                    # This is because we want to be able to optimize the parameters both on the device (through our runtime)
                    # and via the torch optimizers. So this ensures that whichever side updates the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_FATAL @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/ttnn/cpp/ttnn/operations/core/to_layout/to_layout_op.cpp:34: initial_shape[i] == 1
E       info:
E       For ND tensors, shape dimensions greater than 4 should be 1, shape at index0 is 8
E       backtrace:
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x21f7899) [0x7f89029e4899]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x2c9bdee) [0x7f8903488dee]
E        --- tt::tt_metal::Tensor ttnn::operations::core::detail::to_layout_impl<tt::tt_metal::v0::Device>(tt::tt_metal::Tensor const&, tt::tt_metal::Layout, std::optional<tt::tt_metal::DataType> const&, std::optional<tt::tt_metal::MemoryConfig> const&, tt::tt_metal::v0::Device*)
E        --- ttnn::operations::core::ToLayout::invoke(tt::tt_metal::Tensor const&, tt::tt_metal::Layout, std::optional<tt::tt_metal::DataType> const&, std::optional<tt::tt_metal::MemoryConfig> const&, tt::tt_metal::v0::Device*)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7abea5) [0x7f8915818ea5]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7ac976) [0x7f8915819976]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7ad367) [0x7f891581a367]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7aae62) [0x7f8915817e62]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7a96a2) [0x7f89158166a2]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7a4b4b) [0x7f8915811b4b]
E        --- tt::runtime::ttnn::operations::layout::run(tt::target::ttnn::ToLayoutOp const*, tt::runtime::ttnn::ProgramContext&)
E        --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x3106c0) [0x7f8915db16c0]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31060e) [0x7f8915db160e]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf905) [0x7f8915b80905]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x55a478019282]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x55a478008f4d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x55a47800ed14]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x55a478027882]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x55a478003ae8]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]

forge/forge/compiled_graph_state.py:273: RuntimeError

input_shape_and_dim = ((8, 1, 16, 32, 32), -4)

    @pytest.mark.parametrize(
        "input_shape_and_dim",
        [
            ((1, 8, 16, 32, 32), 0),
            ((8, 1, 16, 32, 32), 1),
            ((8, 16, 1, 32, 32), 2),
            ((1, 8, 16, 32, 32), -5),
            ((8, 1, 16, 32, 32), -4),
            ((8, 16, 1, 32, 32), -3),
            ([1, 12, 3200], 0),
            ([1, 1, 2048, 1], [-3, -4]),
            ([1, 64, 1, 1], [-1, -4]),
            ([1, 1, 1, 128], [-2, -4]),
            ([1, 1, 32, 1], [-1, -3]),
            ([1, 1, 1, 64], [-4, -3]),
        ],
    )
    @pytest.mark.push
    def test_squeeze(input_shape_and_dim):
        input_shape, dim = input_shape_and_dim
    
        if input_shape == [1, 12, 3200] or isinstance(dim, list) and len(dim) > 1 and all(d < 0 for d in dim):
            pytest.xfail("TTNN: Tensor layout issues with non tile dim aligned shapes")
    
        class Squeeze(nn.Module):
            def __init__(self):
                super().__init__()
    
            def forward(self, a):
                return torch.squeeze(a, dim)
    
        inputs = [torch.rand(*input_shape)]
    
        framework_model = Squeeze()
        compiled_model = forge.compile(framework_model, sample_inputs=inputs)
    
>       verify(inputs, framework_model, compiled_model)

forge/test/mlir/test_ops.py:1047: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
forge/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f88b33ad570>
inputs = (tensor([[[[[0.49626, 0.76822, 0.08848,  ..., 0.37342, 0.30510, 0.93200],
           [0.17591, 0.26983, 0.15068,  ...,...78639,  ..., 0.73204, 0.66506, 0.97919],
           [0.24259, 0.98793, 0.79220,  ..., 0.74142, 0.07995, 0.86774]]]]]),)
inputs_and_parameters = [tensor([[[[[0.49626, 0.76822, 0.08848,  ..., 0.37342, 0.30510, 0.93200],
           [0.17591, 0.26983, 0.15068,  ...,....78639,  ..., 0.73204, 0.66506, 0.97919],
           [0.24259, 0.98793, 0.79220,  ..., 0.74142, 0.07995, 0.86774]]]]])]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*inputs]
        inputs_and_parameters = [
            *inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        if any([not isinstance(t, torch.Tensor) for t in inputs_and_parameters]):
            logger.info("Converting inputs and parameters to PyTorch tensors...")
            inputs_and_parameters = to_pt_tensors(inputs_and_parameters)
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch module.
                    # This is because we want to be able to optimize the parameters both on the device (through our runtime)
                    # and via the torch optimizers. So this ensures that whichever side updates the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_FATAL @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/ttnn/cpp/ttnn/operations/core/to_layout/to_layout_op.cpp:34: initial_shape[i] == 1
E       info:
E       For ND tensors, shape dimensions greater than 4 should be 1, shape at index0 is 8
E       backtrace:
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x21f7899) [0x7f89029e4899]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x2c9bdee) [0x7f8903488dee]
E        --- tt::tt_metal::Tensor ttnn::operations::core::detail::to_layout_impl<tt::tt_metal::v0::Device>(tt::tt_metal::Tensor const&, tt::tt_metal::Layout, std::optional<tt::tt_metal::DataType> const&, std::optional<tt::tt_metal::MemoryConfig> const&, tt::tt_metal::v0::Device*)
E        --- ttnn::operations::core::ToLayout::invoke(tt::tt_metal::Tensor const&, tt::tt_metal::Layout, std::optional<tt::tt_metal::DataType> const&, std::optional<tt::tt_metal::MemoryConfig> const&, tt::tt_metal::v0::Device*)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7abea5) [0x7f8915818ea5]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7ac976) [0x7f8915819976]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7ad367) [0x7f891581a367]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7aae62) [0x7f8915817e62]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7a96a2) [0x7f89158166a2]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7a4b4b) [0x7f8915811b4b]
E        --- tt::runtime::ttnn::operations::layout::run(tt::target::ttnn::ToLayoutOp const*, tt::runtime::ttnn::ProgramContext&)
E        --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x3106c0) [0x7f8915db16c0]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31060e) [0x7f8915db160e]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf905) [0x7f8915b80905]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x55a478019282]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x55a478008f4d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x55a47800ed14]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x55a478027882]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x55a478003ae8]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]

forge/forge/compiled_graph_state.py:273: RuntimeError

[XPASS(strict)] tt:exception Unsupported dim

[XPASS(strict)] tt:exception Unsupported dim

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] ttnn.embedding op fails while reshaping the input_tensor in TILE_LAYOUT

[XPASS(strict)] tt:exception Unsupported dim

@pytest.mark.push
    def test_mnist_training():
        torch.manual_seed(0)
    
        # Model and data type.
        # For bfloat16, the following line should be added to the test_forge_vs_torch function:
        # In file forge/forge/op/eval/forge/eltwise_unary.py:418 should be replaced with: threshold_tensor = ac.tensor(torch.zeros(shape, dtype=torch.bfloat16) + threshold)
        # That sets relu threshold to bfloat16 tensor.
        # And in file forge/forge/compile.py::compile_main forced bfloat 16 should be added compiler_cfg.default_df_override = DataFormat.Float16_b
        dtype = torch.float32
    
        # Set training hyperparameters
        num_epochs = 3
        batch_size = 2048
        learning_rate = 0.001
    
        # Load dataset
        test_loader, train_loader = load_dataset(batch_size, dtype=dtype)
    
        # Define model and instruct it to compile and run on TT device
        framework_model = MNISTLinear(
            bias=False, dtype=dtype
        )  # bias=False because batch_size=1 with bias=True is not supported
    
        # Create a torch loss and leave on CPU
        loss_fn = torch.nn.CrossEntropyLoss()
    
        # Define optimizer and instruct it to compile and run on TT device
        framework_optimizer = torch.optim.SGD(framework_model.parameters(), lr=learning_rate)
        tt_model = forge.compile(
            framework_model,
            sample_inputs=[torch.rand(batch_size, 784, dtype=dtype)],
            optimizer=framework_optimizer,
            training=True,
        )
    
        logger.info("Starting training loop... (logger will be disabled)")
        logger.disable("")
        for epoch_idx in range(num_epochs):
    
            total_loss = 0
            for batch_idx, (data, target) in enumerate(train_loader):
                # Reset gradients (every batch)
                framework_optimizer.zero_grad()
    
                # Create target tensor and leave on CPU
                target = nn.functional.one_hot(target, num_classes=10).to(dtype)
    
                # Forward pass (prediction) on device
>               pred = tt_model(data)[0]

forge/test/mlir/mnist/training/test_training.py:68: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f882055c4f0>
inputs = (tensor([[-0.42421, -0.42421, -0.42421,  ..., -0.42421, -0.42421, -0.42421],
        [-0.42421, -0.42421, -0.42421,  ....21,  ..., -0.42421, -0.42421, -0.42421],
        [-0.42421, -0.42421, -0.42421,  ..., -0.42421, -0.42421, -0.42421]]),)
inputs_and_parameters = [tensor([[-0.42421, -0.42421, -0.42421,  ..., -0.42421, -0.42421, -0.42421],
        [-0.42421, -0.42421, -0.42421,  ....  0.02915,  0.01170],
        [ 0.00670,  0.03483, -0.01020,  ..., -0.03968, -0.00866, -0.00669]], requires_grad=True)]
name = 'linear_relu_stack.4.weight'
param = Parameter containing:
tensor([[ 0.02984, -0.01652,  0.04047,  ..., -0.03471, -0.02169, -0.00426],
        [ 0.02000, -...,  0.02915,  0.01170],
        [ 0.00670,  0.03483, -0.01020,  ..., -0.03968, -0.00866, -0.00669]], requires_grad=True)

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*inputs]
        inputs_and_parameters = [
            *inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        if any([not isinstance(t, torch.Tensor) for t in inputs_and_parameters]):
            logger.info("Converting inputs and parameters to PyTorch tensors...")
            inputs_and_parameters = to_pt_tensors(inputs_and_parameters)
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch module.
                    # This is because we want to be able to optimize the parameters both on the device (through our runtime)
                    # and via the torch optimizers. So this ensures that whichever side updates the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_THROW @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/program/program.cpp:830: tt::exception
E       info:
E       Statically allocated circular buffers on core range [(x=0,y=0) - (x=3,y=0)] grow to 1671968 B which is beyond max L1 size of 1499136 B
E       backtrace:
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/libtt_metal.so(+0x161ceb) [0x7f891464cceb]
E        --- tt::tt_metal::detail::Program_::validate_circular_buffer_region(tt::tt_metal::v0::Device const*)
E        --- tt::tt_metal::v0::CommandQueue::run_command_impl(tt::tt_metal::CommandInterface const&)
E        --- tt::tt_metal::v0::EnqueueProgram(tt::tt_metal::v0::CommandQueue&, tt::tt_metal::v0::Program&, bool)
E        --- void ttnn::device_operation::detail::launch_on_worker_thread<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >, tt::tt_metal::v0::Device*>(unsigned char, long, tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >&, tt::tt_metal::v0::Device*&)
E        --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::launch_on_single_device<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x2ce4ad9) [0x7f89034d1ad9]
E        --- tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_return_value_t ttnn::device_operation::detail::invoke<tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > > >(unsigned char, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::operation_attributes_t const&, tt::tt_metal::operation::OldInfraDeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >::tensor_args_t const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x2ce13ff) [0x7f89034ce3ff]
E        --- std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > tt::tt_metal::operation::run<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >(tt::tt_metal::operation::DeviceOperation<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >&&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > const&, std::vector<std::optional<tt::tt_metal::Tensor const>, std::allocator<std::optional<tt::tt_metal::Tensor const> > > const&, std::vector<std::optional<tt::tt_metal::Tensor>, std::allocator<std::optional<tt::tt_metal::Tensor> > > const&, unsigned char)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x22edd45) [0x7f8902adad45]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x22ee7ae) [0x7f8902adb7ae]
E        --- ttnn::operations::matmul::matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char, std::optional<tt::tt_metal::Tensor> const&)
E        --- ttnn::operations::matmul::bound_matmul(tt::tt_metal::Tensor const&, tt::tt_metal::Tensor const&, std::optional<tt::tt_metal::Tensor const> const&, ttnn::operations::matmul::Matmul const&, unsigned char const&, std::optional<tt::tt_metal::Tensor>&)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(_ZN4ttnn10operations6matmul15MatmulOperation6invokeERKN2tt8tt_metal6TensorES7_bbRKSt8optionalIKNS4_12MemoryConfigEES8_IKNS4_8DataTypeEERKS8_IKSt7variantIJNS1_28MatmulMultiCoreProgramConfigENS1_45MatmulMultiCoreNonOptimizedReuseProgramConfigENS1_33MatmulMultiCoreReuseProgramConfigENS1_42MatmulMultiCoreReuseMultiCastProgramConfigENS1_44MatmulMultiCoreReuseMultiCast1DProgramConfigENS1_53MatmulMultiCoreReuseMultiCastDRAMShardedProgramConfigEEEERKS8_IKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEES8_IKSH_IJNS_28GrayskullComputeKernelConfigENS_27WormholeComputeKernelConfigEEEES8_IKNS_5types8CoreGridEERKS8_IKNS4_4TileEES8_IS5_E+0x2d1) [0x7f8902a80f01]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7ca913) [0x7f8915837913]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7c9bcd) [0x7f8915836bcd]
E        --- tt::runtime::ttnn::operations::matmul::run(tt::target::ttnn::MatmulOp const*, tt::runtime::ttnn::ProgramContext&)
E        --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x3106c0) [0x7f8915db16c0]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31060e) [0x7f8915db160e]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf905) [0x7f8915b80905]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x55a478019282]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x55a478008f4d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x55a47800ed14]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x55a478008f4d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x55a478003ae8]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]

forge/forge/compiled_graph_state.py:273: RuntimeError

shapes = ((1, 11, 2048), (2048, 128256))

    @pytest.mark.parametrize(
        "shapes",
        [
            ((11, 2048), (2048, 2048)),
            ((1, 32, 1), (1, 1, 11)),
            ((11, 2048), (2048, 512)),
            ((32, 11, 64), (32, 64, 11)),
            ((32, 11, 11), (32, 11, 64)),
            ((11, 2048), (2048, 8192)),
            pytest.param(((1, 11, 8192), (8192, 2048)), marks=pytest.mark.xfail(reason="pcc ~ 0.65")),
            ((1, 11, 2048), (2048, 128256)),
        ],
    )
    @pytest.mark.push
    def test_matmul(shapes):
        shape1, shape2 = shapes
    
        class Matmul(nn.Module):
            def __init__(self):
                super().__init__()
    
            def forward(self, x, y):
                return torch.matmul(x, y)
    
        inputs = [
            torch.rand(shape1),
            torch.rand(shape2),
        ]
    
        framework_model = Matmul()
        compiled_model = forge.compile(framework_model, sample_inputs=inputs)
    
>       verify(inputs, framework_model, compiled_model, VerifyConfig(value_checker=AutomaticValueChecker(pcc=0.95)))

forge/test/mlir/llama/tests/test_specific_ops_llama32.py:164: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
forge/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7f88b33dab30>
inputs = (tensor([[[0.4962565899, 0.7682217956, 0.0884774327,  ..., 0.5874912739, 0.8263171315, 0.2909160852],
         [0.2386...9, 0.2855331302],
        [0.4765424728, 0.8954963684, 0.4173335433,  ..., 0.2985482812, 0.6918529868, 0.8110452890]]))
inputs_and_parameters = [tensor([[[0.4962565899, 0.7682217956, 0.0884774327,  ..., 0.5874912739, 0.8263171315, 0.2909160852],
         [0.2386...9, 0.2855331302],
        [0.4765424728, 0.8954963684, 0.4173335433,  ..., 0.2985482812, 0.6918529868, 0.8110452890]])]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*inputs]
        inputs_and_parameters = [
            *inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        if any([not isinstance(t, torch.Tensor) for t in inputs_and_parameters]):
            logger.info("Converting inputs and parameters to PyTorch tensors...")
            inputs_and_parameters = to_pt_tensors(inputs_and_parameters)
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch module.
                    # This is because we want to be able to optimize the parameters both on the device (through our runtime)
                    # and via the torch optimizers. So this ensures that whichever side updates the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_FATAL @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/tt_metal/impl/dispatch/command_queue.cpp:2416: buffer.num_pages() == 1
E       info:
E       TODO: add support for multi-paged buffer with page size > 64KB
E       backtrace:
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x5e3728) [0x7f8915650728]
E        --- tt::tt_metal::HWCommandQueue::enqueue_write_buffer(tt::tt_metal::v0::Buffer&, void const*, bool, tt::stl::Span<tt::tt_metal::SubDeviceId const, 18446744073709551615ul>)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/libtt_metal.so(_ZN2tt8tt_metal22EnqueueWriteBufferImplERNS0_2v012CommandQueueESt7variantIJSt17reference_wrapperINS1_6BufferEESt10shared_ptrIS6_EEES4_IJKS8_ISt6vectorIhSaIhEEEKS8_ISB_ItSaItEEEKS8_ISB_IiSaIiEEEKS8_ISB_IjSaIjEEEKS8_ISB_IfSaIfEEEKS8_ISB_I8bfloat16SaISW_EEEPKvEEbNS_3stl4SpanIKNS0_11SubDeviceIdELm18446744073709551615EEE+0xce) [0x7f89146893be]
E        --- tt::tt_metal::v0::CommandQueue::run_command_impl(tt::tt_metal::CommandInterface const&)
E        --- tt::tt_metal::v0::EnqueueWriteBuffer(tt::tt_metal::v0::CommandQueue&, std::variant<std::reference_wrapper<tt::tt_metal::v0::Buffer>, std::shared_ptr<tt::tt_metal::v0::Buffer> >, std::variant<std::shared_ptr<std::vector<unsigned char, std::allocator<unsigned char> > > const, std::shared_ptr<std::vector<unsigned short, std::allocator<unsigned short> > > const, std::shared_ptr<std::vector<int, std::allocator<int> > > const, std::shared_ptr<std::vector<unsigned int, std::allocator<unsigned int> > > const, std::shared_ptr<std::vector<float, std::allocator<float> > > const, std::shared_ptr<std::vector<bfloat16, std::allocator<bfloat16> > > const, void const*>)
E        --- void tt::tt_metal::tensor_impl::write_data_to_device_buffer<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::v0::CommandQueue&, tt::tt_metal::borrowed_buffer::Buffer<float> const&, std::shared_ptr<tt::tt_metal::v0::Buffer>, tt::stl::Span<tt::tt_metal::SubDeviceId const, 18446744073709551615ul>)
E        --- std::shared_ptr<tt::tt_metal::v0::Buffer> tt::tt_metal::tensor_impl::initialize_data_on_device<float, tt::tt_metal::borrowed_buffer::Buffer>(tt::tt_metal::borrowed_buffer::Buffer<float>&, tt::tt_metal::v0::Device*, tt::tt_metal::TensorSpec const&, unsigned char, tt::stl::Span<tt::tt_metal::SubDeviceId const, 18446744073709551615ul>)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(_ZSt13__invoke_implISt10shared_ptrIN2tt8tt_metal2v06BufferEEZNS2_11tensor_impl16to_device_bufferIfEES5_RKSt7variantIJNS2_12OwnedStorageENS2_13DeviceStorageENS2_15BorrowedStorageENS2_22MultiDeviceHostStorageENS2_18MultiDeviceStorageEEEPNS3_6DeviceERKNS2_10TensorSpecEhNS1_3stl4SpanIKNS2_11SubDeviceIdELm18446744073709551615EEEEUlOT_E_JRKSB_EESR_St14__invoke_otherOT0_DpOT1_+0x71) [0x7f89027ec0b1]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1ffe869) [0x7f89027eb869]
E        --- tt::tt_metal::Tensor tt::tt_metal::tensor_impl::to_device<float>(tt::tt_metal::Tensor const&, tt::tt_metal::v0::Device*, tt::tt_metal::MemoryConfig const&, unsigned char, tt::stl::Span<tt::tt_metal::SubDeviceId const, 18446744073709551615ul>)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(_ZN2tt8tt_metal11tensor_impl8dispatchIZNS1_17to_device_wrapperIJRNS0_6TensorERPNS0_2v06DeviceERKNS0_12MemoryConfigERhRKSt6vectorINS0_11SubDeviceIdESaISF_EEEEEDaDpOT_EUlTyDpOT0_E_JS5_S9_SC_SD_SJ_EEEDaNS0_8DataTypeEOT_SP_+0xdb) [0x7f89027f2bab]
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal-build/lib/_ttnn.so(+0x1eca202) [0x7f89026b7202]
E        --- tt::tt_metal::tensor_ops::tensor_to(tt::tt_metal::Tensor const&, tt::tt_metal::v0::Device*, tt::tt_metal::MemoryConfig const&, unsigned char, std::vector<tt::tt_metal::SubDeviceId, std::allocator<tt::tt_metal::SubDeviceId> > const&)
E        --- tt::tt_metal::Tensor::to(tt::tt_metal::v0::Device*, tt::tt_metal::MemoryConfig const&, unsigned char, std::vector<tt::tt_metal::SubDeviceId, std::allocator<tt::tt_metal::SubDeviceId> > const&) const
E        --- ttnn::operations::core::to_device(tt::tt_metal::Tensor const&, tt::tt_metal::v0::Device*, std::optional<tt::tt_metal::MemoryConfig> const&, unsigned char, std::vector<tt::tt_metal::SubDeviceId, std::allocator<tt::tt_metal::SubDeviceId> > const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/build/lib/SharedLib/libTTMLIR.so(+0x7a3fc2) [0x7f8915810fc2]
E        --- tt::runtime::ttnn::operations::layout::run(tt::target::ttnn::ToDeviceOp const*, tt::runtime::ttnn::ProgramContext&)
E        --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x3106c0) [0x7f8915db16c0]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0x31060e) [0x7f8915db160e]
E        --- /__w/tt-forge-fe/tt-forge-fe/forge/forge/_C.so(+0xdf905) [0x7f8915b80905]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18b282) [0x55a478019282]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5c7d) [0x55a478008f4d]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x55a47800ed14]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x55a478027882]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x818) [0x55a478003ae8]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55a47802781b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6d2) [0x55a4780039a2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198be1) [0x55a478026be1]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55a47800edbd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55a478023d4c]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29e054) [0x55a47812c054]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55a47800fb4b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x671a) [0x55a4780099ea]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2c89) [0x55a478005f59]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x1a22) [0x55a478004cf2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55a478019aec]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x58aa) [0x55a478008b7a]

forge/forge/compiled_graph_state.py:273: RuntimeError