record_forge_property = <test.conftest.ForgePropertyStore object at 0x7fa6c5ff9db0>
variant = 'xception'

    @pytest.mark.push
    @pytest.mark.nightly
    @pytest.mark.parametrize("variant", variants, ids=variants)
    def test_xception_timm(record_forge_property, variant):
        if variant not in ["xception", "xception71.tf_in1k"]:
            pytest.skip("Skipping due to the current CI/CD pipeline limitations")
    
        # Build Module Name
        module_name = build_module_name(
            framework=Framework.PYTORCH,
            model="xception",
            variant=variant,
            source=Source.TIMM,
            task=Task.IMAGE_CLASSIFICATION,
        )
    
        # Record Forge Property
        record_forge_property("model_name", module_name)
    
        (framework_model, inputs) = generate_model_xception_imgcls_timm(variant)
    
        # Forge compile framework model
        compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name)
    
        # Model Verification
>       verify(inputs, framework_model, compiled_model)

forge/test/models/pytorch/vision/xception/test_xception.py:67: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/verify/verify.py:302: in verify
    co_out = compiled_model(*inputs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <forge.compiled_graph_state.CompiledModel object at 0x7fa6c16f9f60>
inputs = (tensor([[[[-0.90588, -0.90588, -0.91373,  ..., -0.96078, -0.96863, -0.96863],
          [-0.91373, -0.92157, -0.92157... ..., -0.76471, -0.77255, -0.89804],
          [-0.60784, -0.65490, -0.77255,  ..., -0.77255, -0.69412, -0.88235]]]]),)
inputs_and_parameters = [tensor([[[[-0.90588, -0.90588, -0.91373,  ..., -0.96078, -0.96863, -0.96863],
          [-0.91373, -0.92157, -0.92157...0.71037, 0.78038, 0.65737, 0.82360, 0.67960, 0.59922, 0.65910, 0.91053, 0.72178, 0.84982]]]], requires_grad=True), ...]

    def __call__(self, *inputs: AnyTensor) -> List[torch.Tensor]:
        """
        Run inference on the compiled model.
    
        Parameters
        ----------
        inputs: [Tensor, ...]
            Input tensors
    
        Returns
        -------
        List[Tensor]
            Output tensors
        """
        self.inputs = [*to_pt_tensors(inputs)]
    
        inputs_and_parameters = [
            *self.inputs,
            *self.fwd_compiled_graph_state.get_ordered_constant_tensors(),
            *self.fwd_compiled_graph_state.get_ordered_parameter_tensors(),
        ]
    
        assert all(
            [isinstance(t, torch.Tensor) for t in inputs_and_parameters]
        ), "All inputs should be torch tensors by now."
    
        if self.training() and isinstance(self.framework_module, PyTorchModule):
            for name, param in self.framework_module.module.named_parameters():
                if param.requires_grad:
                    our_tensor = self.fwd_compiled_graph_state.get_parameter_tensor(name)
    
                    # NOTE: for parameters that require gradients, we want to share the same tensor with the PyTorch
                    # module. This is because we want to be able to optimize the parameters both on the device
                    # (through our runtime) and via the torch optimizers. So this ensures that whichever side updates
                    # the parameter value, the other side can see the change.
                    #
                    # This could change in the future, but for now ensure that our premise is correct.
                    assert param is our_tensor
    
        logger.info(
            f"Running model {self.framework_module.get_name()} {self.fwd_compiled_graph_state.graph.get_name()} on device..."
        )
>       all_outputs = run_binary(self.compiled_binary, int(ProgramId.FORWARD), inputs_and_parameters)
E       RuntimeError: TT_FATAL @ /__w/tt-forge-fe/tt-forge-fe/third_party/tt-mlir/third_party/tt-metal/src/tt-metal/ttnn/cpp/ttnn/operations/pool/generic/device/pool_op.cpp:37: (input_shape[3] % tt::constants::TILE_WIDTH == 0) || (input_shape[3] == 16)
E       info:
E       Input channels (728) should be padded to nearest TILE_WIDTH (32) or should be 16
E       backtrace:
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libtt_metal.so(+0xfc859) [0x7fa62184c859]
E        --- ttnn::operations::pool::validate_pool2d(tt::tt_metal::Tensor const&, ttnn::operations::sliding_window::SlidingWindowConfig const&, tt::tt_metal::MemoryConfig const&)
E        --- void ttnn::device_operation::detail::launch_on_worker_thread<ttnn::operations::pool::Pool2D, tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>, long, ttnn::operations::pool::Pool2D::operation_attributes_t, ttnn::operations::pool::Pool2D::tensor_args_t, tt::tt_metal::Tensor, tt::tt_metal::v0::IDevice*>(tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>, long, ttnn::operations::pool::Pool2D::operation_attributes_t const&, ttnn::operations::pool::Pool2D::tensor_args_t const&, tt::tt_metal::Tensor&, tt::tt_metal::v0::IDevice*&)
E        --- ttnn::operations::pool::Pool2D::tensor_return_value_t ttnn::device_operation::detail::launch_on_single_device<ttnn::operations::pool::Pool2D>(tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>, ttnn::operations::pool::Pool2D::operation_attributes_t const&, ttnn::operations::pool::Pool2D::tensor_args_t const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0xf6c657) [0x7fa61f115657]
E        --- ttnn::operations::pool::Pool2D::tensor_return_value_t ttnn::device_operation::detail::invoke<ttnn::operations::pool::Pool2D>(tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>, ttnn::operations::pool::Pool2D::operation_attributes_t const&, ttnn::operations::pool::Pool2D::tensor_args_t const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0xf6c0df) [0x7fa61f1150df]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0xf6bca9) [0x7fa61f114ca9]
E        --- ttnn::operations::pool::Pool2DOp<(ttnn::operations::pool::Pool2DType)0>::invoke(tt::stl::StrongType<unsigned char, ttnn::QueueIdTag>, tt::tt_metal::Tensor const&, unsigned int, unsigned int, unsigned int, unsigned int, std::array<unsigned int, 2ul>, std::array<unsigned int, 2ul>, std::array<unsigned int, 2ul>, std::array<unsigned int, 2ul>, std::optional<tt::tt_metal::MemoryConfig const> const&, std::optional<tt::tt_metal::TensorMemoryLayout const>, bool)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15c2c6) [0x7fa6306eb2c6]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15be5b) [0x7fa6306eae5b]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15bd1c) [0x7fa6306ead1c]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x25c6fe2) [0x7fa62076ffe2]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/_ttnn.so(+0x25c77cc) [0x7fa6207707cc]
E        --- void tt::tt_metal::operation::launch_op_func<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > >(std::function<std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > (std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> > const&, std::vector<std::optional<tt::tt_metal::Tensor const>, std::allocator<std::optional<tt::tt_metal::Tensor const> > > const&, std::vector<std::optional<tt::tt_metal::Tensor>, std::allocator<std::optional<tt::tt_metal::Tensor> > > const&)> const&, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >, std::vector<tt::tt_metal::Tensor, std::allocator<tt::tt_metal::Tensor> >&, std::vector<std::optional<tt::tt_metal::Tensor const>, std::allocator<std::optional<tt::tt_metal::Tensor const> > >, std::vector<std::optional<tt::tt_metal::Tensor>, std::allocator<std::optional<tt::tt_metal::Tensor> > >, bool)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15bb13) [0x7fa6306eab13]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15b588) [0x7fa6306ea588]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/lib/libTTMLIRRuntime.so(+0x15b073) [0x7fa6306ea073]
E        --- tt::runtime::ttnn::operations::pool::run(tt::target::ttnn::MaxPool2dOp const*, tt::runtime::ttnn::ProgramContext&)
E        --- tt::runtime::ttnn::runProgram(tt::tt_metal::distributed::MeshDevice&, tt::runtime::Binary, unsigned int, std::vector<tt::tt_metal::Tensor*, std::allocator<tt::tt_metal::Tensor*> > const&)
E        --- tt::runtime::ttnn::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::runtime::submit(tt::runtime::Device, tt::runtime::Binary, unsigned int, std::vector<tt::runtime::Tensor, std::allocator<tt::runtime::Tensor> > const&)
E        --- tt::run_binary(tt::runtime::Binary&, int, std::vector<at::Tensor, std::allocator<at::Tensor> > const&)
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x316d40) [0x7fa630ab7d40]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0x316c8e) [0x7fa630ab7c8e]
E        --- /opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/_C.so(+0xe0d75) [0x7fa630881d75]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x18ae12) [0x55cfbee48e12]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55cfbee3f75b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x59d7) [0x55cfbee38c47]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0xc4) [0x55cfbee3e934]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55cfbee538ec]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29cba4) [0x55cfbef5aba4]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55cfbee573cb]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6bf) [0x55cfbee3392f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0x122) [0x55cfbee57432]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198791) [0x55cfbee56791]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55cfbee3e9dd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55cfbee538ec]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29cba4) [0x55cfbef5aba4]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55cfbee3f75b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x66f2) [0x55cfbee39962]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x804) [0x55cfbee33a74]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198791) [0x55cfbee56791]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55cfbee3e9dd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55cfbee538ec]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29cba4) [0x55cfbef5aba4]
E        --- /opt/ttforge-toolchain/venv/bin/python(PyObject_Call+0xbb) [0x55cfbee573cb]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6bf) [0x55cfbee3392f]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198791) [0x55cfbee56791]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x6bf) [0x55cfbee3392f]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x198791) [0x55cfbee56791]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x5642) [0x55cfbee388b2]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_FastCallDictTstate+0x16d) [0x55cfbee3e9dd]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_Call_Prepend+0x5c) [0x55cfbee538ec]
E        --- /opt/ttforge-toolchain/venv/bin/python(+0x29cba4) [0x55cfbef5aba4]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyObject_MakeTpCall+0x25b) [0x55cfbee3f75b]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x66f2) [0x55cfbee39962]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x2a83) [0x55cfbee35cf3]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyFunction_Vectorcall+0x7c) [0x55cfbee4966c]
E        --- /opt/ttforge-toolchain/venv/bin/python(_PyEval_EvalFrameDefault+0x18d3) [0x55cfbee34b43]

/opt/ttforge-toolchain/venv/lib/python3.10/site-packages/forge/compiled_graph_state.py:254: RuntimeError