Skip to content
On PR

[tests] add loss tests to CI #1829

Sign in to view logs

[tests] add loss tests to CI

[tests] add loss tests to CI #1829

GitHub Actions / TT-Forge-FE Tests failed Dec 26, 2024 in 0s

763 tests run, 483 passed, 273 skipped, 7 failed.

Annotations

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape1] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.02277, grad_fn=<MeanBackward0>), tensor([0.]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (128,), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.02277, grad_fn=<MeanBackward0>), tensor([0.]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape3] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.15346, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (32, 32), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.15346, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape5] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.12397, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (128, 20), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1.12397, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape1] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(130.91412, grad_fn=<SumBackward0>), tensor([0.]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (128,), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(130.91412, grad_fn=<SumBackward0>), tensor([0.]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape3] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1181.14148, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (32, 32), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(1181.14148, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape5] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(2877.35303, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (128, 20), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(2877.35303, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 65 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_cross_entropy_loss[prediction_shape1] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(3.90286, grad_fn=<DivBackward1>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose
Raw output 
prediction_shape = (32, 32)

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            pytest.param((3, 5), marks=pytest.mark.xfail(reason="incorrect result in ttnn")),
            (32, 32),
            pytest.param((33, 127), marks=pytest.mark.xfail(reason="incorrect result in ttnn")),
            pytest.param((128, 20), marks=pytest.mark.xfail(reason="incorrect result in ttnn")),
        ],
    )
    @pytest.mark.push
    def test_cross_entropy_loss(prediction_shape):
        forge_loss = forge.op.loss.CrossEntropyLoss("cross_entropy_loss")
        torch_loss = torch.nn.CrossEntropyLoss()
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.empty(prediction_shape[0], dtype=torch.long).random_(prediction_shape[-1])
        target = nn.functional.one_hot(target, num_classes=prediction_shape[-1]).float()
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff63c696480>(tensor(3.90286, grad_fn=<DivBackward1>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff63c696480> = torch.allclose

forge/test/mlir/test_loss.py:65: AssertionError
View more details on GitHub Actions