change "nightly" to "op_by_op"

.github/workflows/run-model-tests.yml

@@ -156,7 +156,7 @@ jobs:
         echo ${{ steps.strings.outputs.install-output-dir }}
         source env/activate
         export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}"
-        pytest -svv --nightly ${{ matrix.build.tests }}
+        pytest -svv --op_by_op ${{ matrix.build.tests }}
 
     - name: Tar results
       if: success() || failure()

.github/workflows/run-tests.yml

@@ -108,23 +108,23 @@ jobs:
         source env/activate
         export LD_LIBRARY_PATH="/opt/ttmlir-toolchain/lib/:${{ steps.strings.outputs.install-output-dir }}/lib:${{ steps.strings.outputs.build-output-dir }}/lib:./lib/:${LD_LIBRARY_PATH}"
         pytest -v \
-                  tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[push-eval] \
-                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-base-patch16-224-eval] \
-                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[push-microsoft/beit-large-patch16-224-eval] \
-                  tests/models/detr/test_detr.py::test_detr[push-eval] \
-                  tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[push-eval] \
-                  tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[push-eval] \
-                  tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[push-eval] \
-                  tests/models/mnist/test_mnist.py::test_mnist_train[push-eval] \
-                  tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[push-eval] \
-                  tests/models/openpose/test_openpose_v2.py::test_openpose_v2[push-eval] \
-                  tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[push-eval] \
-                  tests/models/resnet/test_resnet.py::test_resnet[push-eval] \
-                  tests/models/resnet50/test_resnet50.py::test_resnet[push-eval] \
-                  tests/models/segformer/test_segformer.py::test_segformer[push-eval] \
-                  tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[push-eval] \
-                  tests/models/vilt/test_vilt.py::test_vilt[push-eval] \
-                  tests/models/yolov3/test_yolov3.py::test_yolov3[push-eval] \
+                  tests/models/autoencoder_linear/test_autoencoder_linear.py::test_autoencoder_linear[full-eval] \
+                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[full-microsoft/beit-base-patch16-224-eval] \
+                  tests/models/beit/test_beit_image_classification.py::test_beit_image_classification[full-microsoft/beit-large-patch16-224-eval] \
+                  tests/models/detr/test_detr.py::test_detr[full-eval] \
+                  tests/models/glpn_kitti/test_glpn_kitti.py::test_glpn_kitti[full-eval] \
+                  tests/models/mgp-str-base/test_mgp_str_base.py::test_mgp_str_base[full-eval] \
+                  tests/models/mlpmixer/test_mlpmixer.py::test_mlpmixer[full-eval] \
+                  tests/models/mnist/test_mnist.py::test_mnist_train[full-eval] \
+                  tests/models/MobileNetV2/test_MobileNetV2.py::test_MobileNetV2[full-eval] \
+                  tests/models/openpose/test_openpose_v2.py::test_openpose_v2[full-eval] \
+                  tests/models/perceiver_io/test_perceiver_io.py::test_perceiver_io[full-eval] \
+                  tests/models/resnet/test_resnet.py::test_resnet[full-eval] \
+                  tests/models/resnet50/test_resnet50.py::test_resnet[full-eval] \
+                  tests/models/segformer/test_segformer.py::test_segformer[full-eval] \
+                  tests/models/squeeze_bert/test_squeeze_bert.py::test_squeeze_bert[full-eval] \
+                  tests/models/vilt/test_vilt.py::test_vilt[full-eval] \
+                  tests/models/yolov3/test_yolov3.py::test_yolov3[full-eval] \
            --junit-xml=${{ steps.strings.outputs.test_report_path_models }}
 
     - name: Upload Test Report Models

tests/conftest.py

@@ -15,37 +15,40 @@ def run_around_tests():
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--nightly", action="store_true", default=False, help="Run nightly tests"
+        "--op_by_op",
+        action="store_true",
+        default=False,
+        help="Run test in op-by-op mode",
     )
 
 
 def pytest_collection_modifyitems(config, items):
-    # If --nightly flag is set, filter out tests with nightly=False
+    # If --op_by_op flag is set, filter out tests with op_by_op=False
     selected_items = []
     for item in items:
-        # Check if the test has a parameter called 'nightly'
+        # Check if the test has a parameter called 'op_by_op'
         # and whether it is set to True
 
-        if config.getoption("--nightly"):
+        if config.getoption("--op_by_op"):
             for param in item.iter_markers(name="parametrize"):
-                # Check if the parameter is 'nightly' and its value is True
-                if "nightly" in param.args[0] and item.callspec.params["nightly"]:
+                # Check if the parameter is 'op_by_op' and its value is True
+                if "op_by_op" in param.args[0] and item.callspec.params["op_by_op"]:
                     selected_items.append(item)
                     break
         else:
-            # If the test does not have a 'nightly' parameter,
-            # add all tests without a nightly parameter, as well
-            # as all testst with a nightly parameter where nightly=False
-            has_nightly_param = False
+            # If the test does not have a 'op_by_op' parameter,
+            # add all tests without a op_by_op parameter, as well
+            # as all testst with a op_by_op parameter where op_by_op=False
+            has_op_by_op_param = False
             for param in item.iter_markers(name="parametrize"):
-                if "nightly" in param.args[0]:
-                    has_nightly_param = True
-                    # Only add the test if nightly=False
-                    if not item.callspec.params["nightly"]:
+                if "op_by_op" in param.args[0]:
+                    has_op_by_op_param = True
+                    # Only add the test if op_by_op=False
+                    if not item.callspec.params["op_by_op"]:
                         selected_items.append(item)
                         break
             # If theres no nigtly parameter, add the test
-            if not has_nightly_param:
+            if not has_op_by_op_param:
                 selected_items.append(item)
-        # Replace the items with only the nightly tests
+        # Replace the items with only the op_by_op tests
     items[:] = selected_items

tests/models/MobileNetV2/test_MobileNetV2.py

@@ -35,15 +35,15 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_MobileNetV2(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_MobileNetV2(record_property, mode, op_by_op):
     model_name = "MobileNetV2"
     record_property("model_name", model_name)
     record_property("mode", mode)
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/Qwen/test_qwen2_casual_lm.py

@@ -42,14 +42,14 @@ def set_model_eval(self, model):
         "Qwen/Qwen2.5-1.5B",
     ],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_qwen2_casual_lm(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_qwen2_casual_lm(record_property, model_name, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     record_property("model_name", model_name)
     record_property("mode", mode)
     cc = CompilerConfig()
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, compiler_config=cc)

tests/models/Qwen/test_qwen2_token_classification.py

@@ -35,15 +35,15 @@ def _load_inputs(self):
         "Qwen/Qwen2-7B",
     ],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_qwen2_token_classification(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_qwen2_token_classification(record_property, model_name, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/RMBG/test_RMBG.py

@@ -40,8 +40,8 @@ def _load_inputs(self):
     ["train", "eval"],
 )
 @pytest.mark.xfail(reason="Fails due pt2 compile issue, graph is traced")
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_RMBG(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_RMBG(record_property, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     model_name = "RMBG"
@@ -51,7 +51,7 @@ def test_RMBG(record_property, mode, nightly):
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, compiler_config=cc)

tests/models/albert/test_albert_masked_lm.py

@@ -48,15 +48,15 @@ def append_fake_loss_function(self, outputs):
         "albert/albert-xxlarge-v2",
     ],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_albert_masked_lm(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_albert_masked_lm(record_property, model_name, mode, op_by_op):
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/albert/test_albert_question_answering.py

@@ -31,15 +31,15 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["twmkn9/albert-base-v2-squad2"])
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_albert_question_answering(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_albert_question_answering(record_property, model_name, mode, op_by_op):
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, relative_atol=0.01, compiler_config=cc)

tests/models/albert/test_albert_sequence_classification.py

@@ -31,15 +31,15 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["textattack/albert-base-v2-imdb"])
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_albert_sequence_classification(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_albert_sequence_classification(record_property, model_name, mode, op_by_op):
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/albert/test_albert_token_classification.py

@@ -33,15 +33,15 @@ def _load_inputs(self):
     ["eval"],
 )
 @pytest.mark.parametrize("model_name", ["albert/albert-base-v2"])
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_albert_token_classification(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_albert_token_classification(record_property, model_name, mode, op_by_op):
     record_property("model_name", f"{model_name}-classification")
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/autoencoder_conv/test_autoencoder_conv.py

@@ -50,16 +50,16 @@ def _load_inputs(self):
     ["train", "eval"],
 )
 @pytest.mark.skip(reason="PyTorch compilation flow cannot accept pipeline.")
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_autoencoder_conv(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_autoencoder_conv(record_property, mode, op_by_op):
     model_name = "Autoencoder (convolutional)"
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, compiler_config=cc)

tests/models/autoencoder_conv/test_autoencoder_conv_v2.py

@@ -75,16 +75,16 @@ def _load_inputs(self):
     "mode",
     ["train", "eval"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_autoencoder_conv_v2(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_autoencoder_conv_v2(record_property, mode, op_by_op):
     model_name = f"Autoencoder (conv)"
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, compiler_config=cc)

tests/models/autoencoder_linear/test_autoencoder_linear.py

@@ -84,8 +84,8 @@ def _load_inputs(self):
     "mode",
     ["train", "eval"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_autoencoder_linear(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_autoencoder_linear(record_property, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     model_name = "Autoencoder (linear)"
@@ -95,7 +95,7 @@ def test_autoencoder_linear(record_property, mode, nightly):
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, compiler_config=cc)

tests/models/beit/test_beit_image_classification.py

@@ -43,8 +43,8 @@ def get_results_train(self, model, inputs, outputs):
     "model_name",
     ["microsoft/beit-base-patch16-224", "microsoft/beit-large-patch16-224"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_beit_image_classification(record_property, model_name, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_beit_image_classification(record_property, model_name, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     record_property("model_name", model_name)
@@ -53,7 +53,7 @@ def test_beit_image_classification(record_property, model_name, mode, nightly):
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
     else:
         cc.compile_depth = CompileDepth.TTNN_IR

tests/models/bert/test_bert.py

@@ -42,16 +42,16 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_bert(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_bert(record_property, mode, op_by_op):
     model_name = "BERT"
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(model_name, mode, relative_atol=0.012, compiler_config=cc)

tests/models/bloom/test_bloom.py

@@ -38,16 +38,16 @@ def _load_inputs(self):
     "mode",
     ["eval"],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_bloom(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_bloom(record_property, mode, op_by_op):
     model_name = "Bloom"
     record_property("model_name", model_name)
     record_property("mode", mode)
 
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(

tests/models/clip/test_clip.py

@@ -59,8 +59,8 @@ def get_results_train(self, model, inputs, outputs):
         "eval",
     ],
 )
-@pytest.mark.parametrize("nightly", [True, False], ids=["nightly", "push"])
-def test_clip(record_property, mode, nightly):
+@pytest.mark.parametrize("op_by_op", [True, False], ids=["op_by_op", "full"])
+def test_clip(record_property, mode, op_by_op):
     if mode == "train":
         pytest.skip()
     model_name = "CLIP"
@@ -70,7 +70,7 @@ def test_clip(record_property, mode, nightly):
     cc = CompilerConfig()
     cc.enable_consteval = True
     cc.consteval_parameters = True
-    if nightly:
+    if op_by_op:
         cc.compile_depth = CompileDepth.EXECUTE_OP_BY_OP
 
     tester = ThisTester(