Skip to content
On PR

[tests] add loss tests to CI #1829

Sign in to view logs

[tests] add loss tests to CI

[tests] add loss tests to CI #1829

GitHub Actions / TT-Forge-FE Tests failed Dec 26, 2024 in 0s

700 tests run, 419 passed, 275 skipped, 6 failed.

Annotations

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape0] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.21405, grad_fn=<MeanBackward0>), tensor([0.]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (33,), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.21405, grad_fn=<MeanBackward0>), tensor([0.]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape2] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.06760, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (3, 5), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.06760, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[mean-prediction_shape4] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.14280, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (33, 127), reduction = 'mean'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(1.14280, grad_fn=<MeanBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape0] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(40.06364, grad_fn=<SumBackward0>), tensor([0.]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (33,), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(40.06364, grad_fn=<SumBackward0>), tensor([0.]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape2] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(16.01401, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (3, 5), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(16.01401, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError

Check failure on line 38 in forge/test/mlir/test_loss.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_loss.test_l1_loss[sum-prediction_shape4] 

assert False
 +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(4789.45557, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
 +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose
Raw output 
prediction_shape = (33, 127), reduction = 'sum'

    @pytest.mark.parametrize(
        "prediction_shape",
        [
            (33,),
            (128,),
            (3, 5),
            (32, 32),
            (33, 127),
            (128, 20),
        ],
    )
    @pytest.mark.parametrize("reduction", ["mean", "sum"])
    @pytest.mark.push
    def test_l1_loss(prediction_shape, reduction):
        forge_loss = forge.op.loss.L1Loss("l1_loss", reduction=reduction)
        torch_loss = torch.nn.L1Loss(reduction=reduction)
    
        prediction = torch.randn(prediction_shape, requires_grad=True)
        prediction_forge = forge.tensor.Tensor.create_from_torch(prediction)
        target = torch.randn((prediction_shape))
        target_forge = forge.tensor.Tensor.create_from_torch(target)
    
        forge_loss = forge.compile(forge_loss, sample_inputs=[prediction_forge, target_forge])
        forge_loss_out = forge_loss(prediction, target)
        torch_loss_out = torch_loss(prediction, target)
    
>       assert torch.allclose(torch_loss_out, forge_loss_out[0], rtol=11e-3)
E       assert False
E        +  where False = <built-in method allclose of type object at 0x7ff8c4096480>(tensor(4789.45557, grad_fn=<SumBackward0>), tensor([[0.]]), rtol=0.011)
E        +    where <built-in method allclose of type object at 0x7ff8c4096480> = torch.allclose

forge/test/mlir/test_loss.py:38: AssertionError
View more details on GitHub Actions