Skip to content

Clean nightly model runs for cleaner tracking of high priority models. #2671

Clean nightly model runs for cleaner tracking of high priority models.

Clean nightly model runs for cleaner tracking of high priority models. #2671

GitHub Actions / TT-Forge-FE Tests failed Feb 22, 2025 in 0s

663 tests run, 525 passed, 137 skipped, 1 failed.

Annotations

Check failure on line 72 in forge/test/models/pytorch/vision/densenet/test_densenet.py

See this annotation in the file changed.

@github-actions github-actions / TT-Forge-FE Tests

test_densenet.test_densenet_121_pytorch[densenet121_hf_xray]

ValueError: Data mismatch -> AutomaticValueChecker (compare_with_golden): framework_model=tensor([[0.04869, 0.03288, 0.15531, 0.00566, 0.00114, 0.00461, 0.04744, 0.05742, 0.02110, 0.01312, 0.03669, 0.39452, 0.29256, 0.00085, 0.00119, 0.07292, 0.34686, 0.02730]], grad_fn=<SigmoidBackward0>), compiled_model=tensor([[0.02852, 0.02478, 0.21812, 0.00000, 0.00000, 0.00000, 0.05164, 0.04782, 0.01900, 0.02173, 0.02818, 0.42412, 0.28394, 0.00000, 0.00000, 0.04551, 0.45030, 0.02611]])
Raw output
record_forge_property = <function record_property.<locals>.append_property at 0x7f9837e1c700>
variant = 'densenet121_hf_xray'

    @pytest.mark.push
    @pytest.mark.nightly
    @pytest.mark.parametrize("variant", variants, ids=variants)
    def test_densenet_121_pytorch(record_forge_property, variant):
        if variant == "densenet121":
            pytest.skip("Skipping due to the current CI/CD pipeline limitations")
    
        # Build Module Name
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="densenet",
            variant=variant,
            source=Source.TORCHVISION,
            task=Task.IMAGE_CLASSIFICATION,
        )
    
        # Record Forge Property
        record_forge_property("model_name", module_name)
    
        # STEP 2: Create Forge module from PyTorch model
        if variant == "densenet121":
            framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", "densenet121", pretrained=True)
            img_tensor = get_input_img()
        else:
            model_name = "densenet121-res224-all"
            model = download_model(xrv.models.get_model, model_name)
            framework_model = densenet_xray_wrapper(model)
            img_tensor = get_input_img_hf_xray()
    
        # STEP 3: Run inference on Tenstorrent device
        inputs = [img_tensor]
    
        # Forge compile framework model
        compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
    
        # Model Verification
>       verify(inputs, framework_model, compiled_model)

forge/test/models/pytorch/vision/densenet/test_densenet.py:72: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:333: in verify
    verify_cfg.value_checker.check(fw, co)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.verify.value_checkers.AutomaticValueChecker object at 0x7f9875658a30>
fw_out = tensor([[0.04869, 0.03288, 0.15531, 0.00566, 0.00114, 0.00461, 0.04744, 0.05742, 0.02110, 0.01312, 0.03669, 0.39452, 0.29256, 0.00085, 0.00119, 0.07292, 0.34686, 0.02730]], grad_fn=<SigmoidBackward0>)
co_out = tensor([[0.02852, 0.02478, 0.21812, 0.00000, 0.00000, 0.00000, 0.05164, 0.04782, 0.01900, 0.02173, 0.02818, 0.42412, 0.28394, 0.00000, 0.00000, 0.04551, 0.45030, 0.02611]])

    def check(self, fw_out, co_out):
        if not compare_with_golden(fw_out, co_out, self.pcc, self.rtol, self.atol, self.dissimilarity_threshold):
>           raise ValueError(
                f"Data mismatch -> AutomaticValueChecker (compare_with_golden): framework_model={fw_out}, compiled_model={co_out}"
            )
E           ValueError: Data mismatch -> AutomaticValueChecker (compare_with_golden): framework_model=tensor([[0.04869, 0.03288, 0.15531, 0.00566, 0.00114, 0.00461, 0.04744, 0.05742, 0.02110, 0.01312, 0.03669, 0.39452, 0.29256, 0.00085, 0.00119, 0.07292, 0.34686, 0.02730]], grad_fn=<SigmoidBackward0>), compiled_model=tensor([[0.02852, 0.02478, 0.21812, 0.00000, 0.00000, 0.00000, 0.05164, 0.04782, 0.01900, 0.02173, 0.02818, 0.42412, 0.28394, 0.00000, 0.00000, 0.04551, 0.45030, 0.02611]])

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/value_checkers.py:38: ValueError