From 3e33ee3116ece73212bfeb38652cfa4b361a1532 Mon Sep 17 00:00:00 2001
From: Yunchu Lee <yunchu.lee@intel.com>
Date: Tue, 13 Aug 2024 08:03:15 +0900
Subject: [PATCH] Update fuzzing (#3829)

* update-fuzzing-tests

* extend available torch version
---
 pyproject.toml                         | 12 +++++++-----
 src/otx/cli/utils/installation.py      |  7 +++++++
 tests/fuzzing/assets/cli/commands.dict |  3 +++
 tests/fuzzing/cli_fuzzing.py           |  6 ++++--
 tests/fuzzing/eval_fuzzing_crash.py    | 26 ++++++++++++++++++++++++++
 tox.ini                                |  8 +++++++-
 6 files changed, 54 insertions(+), 8 deletions(-)
 mode change 100644 => 100755 tests/fuzzing/cli_fuzzing.py
 create mode 100644 tests/fuzzing/eval_fuzzing_crash.py

diff --git a/pyproject.toml b/pyproject.toml
index 395aa8ae63c..6159b79d92f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -156,11 +156,6 @@ include = ["otx*"]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # COVERAGE CONFIGURATION.                                                     #
-[tool.coverage.paths]
-source = [
-    "src",
-]
-
 [tool.coverage.report]
 exclude_lines = [
     "pragma: no cover",
@@ -168,6 +163,9 @@ exclude_lines = [
 ]
 
 [tool.coverage.run]
+source = [
+    "src/otx/",
+]
 omit = [
     "**/__init__.py",
     "src/otx/recipes/*",
@@ -184,6 +182,10 @@ omit = [
     "src/otx/core/data/transform_libs/mmseg.py",
     "src/otx/core/exporter/mmdeploy.py",
     "src/otx/core/model/utils/*",
+
+    # Ignore some generated files by opencv-python
+    "config.py",
+    "config-3.py",
 ]
 
 
diff --git a/src/otx/cli/utils/installation.py b/src/otx/cli/utils/installation.py
index b79dc53ff12..0a554c3a3b3 100644
--- a/src/otx/cli/utils/installation.py
+++ b/src/otx/cli/utils/installation.py
@@ -25,6 +25,13 @@
     "2.0.0": {"torchvision": "0.15.1", "cuda": ("11.7", "11.8")},
     "2.0.1": {"torchvision": "0.15.2", "cuda": ("11.7", "11.8")},
     "2.1.1": {"torchvision": "0.16.1", "cuda": ("11.8", "12.1")},
+    "2.1.2": {"torchvision": "0.16.2", "cuda": ("11.8", "12.1")},
+    "2.2.0": {"torchvision": "0.17.0", "cuda": ("11.8", "12.1")},
+    "2.2.1": {"torchvision": "0.17.1", "cuda": ("11.8", "12.1")},
+    "2.2.2": {"torchvision": "0.17.2", "cuda": ("11.8", "12.1")},
+    "2.3.0": {"torchvision": "0.18.0", "cuda": ("11.8", "12.1")},
+    "2.3.1": {"torchvision": "0.18.1", "cuda": ("11.8", "12.1")},
+    "2.4.0": {"torchvision": "0.19.0", "cuda": ("11.8", "12.1", "12.4")},
 }
 
 MM_REQUIREMENTS = [
diff --git a/tests/fuzzing/assets/cli/commands.dict b/tests/fuzzing/assets/cli/commands.dict
index 652e4fba745..cd021623a67 100644
--- a/tests/fuzzing/assets/cli/commands.dict
+++ b/tests/fuzzing/assets/cli/commands.dict
@@ -1,3 +1,5 @@
+"-h"
+"-v"
 "install"
 "find"
 "train"
@@ -6,3 +8,4 @@
 "export"
 "optimize"
 "explain"
+"benchmark"
diff --git a/tests/fuzzing/cli_fuzzing.py b/tests/fuzzing/cli_fuzzing.py
old mode 100644
new mode 100755
index 788c223c3e4..85ad915619d
--- a/tests/fuzzing/cli_fuzzing.py
+++ b/tests/fuzzing/cli_fuzzing.py
@@ -1,3 +1,5 @@
+#!.tox/fuzzing/bin/python
+
 import sys
 
 import atheris
@@ -17,8 +19,8 @@ def fuzz_otx(input_bytes):
     try:
         _ = cli_main()
     except SystemExit as e:
-        # argparser will throw SystemExit with code 2 when some required arguments are missing
-        if e.code != 2:
+        # argparser will throw SystemExit with code 0 or 2 when completed successfuly or some required arguments are missing
+        if e.code not in [0, 2]:
             raise
     finally:
         sys.argv = backup_argv
diff --git a/tests/fuzzing/eval_fuzzing_crash.py b/tests/fuzzing/eval_fuzzing_crash.py
new file mode 100644
index 00000000000..f44832aaf28
--- /dev/null
+++ b/tests/fuzzing/eval_fuzzing_crash.py
@@ -0,0 +1,26 @@
+#!.tox/fuzzing/bin/python
+
+import sys
+from pathlib import Path
+
+from otx.cli import main as cli_main
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: python eval_fuzzing_crash.py <fuzzing-crash-binary-file>")
+        return
+
+    ba = None
+    if Path.exists(sys.argv[1]):
+        with Path.open(sys.argv[1], "rb") as f:
+            ba = bytearray(f.read())
+
+    arguments = "".join(ba.decode(errors="replace"))
+
+    sys.argv = ["otx", f"{arguments!s}"]
+    _ = cli_main()
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tox.ini b/tox.ini
index 3bdff28a4c3..83fe336bfa0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -105,7 +105,13 @@ deps =
     atheris
     coverage
 extras = full
+commands_pre =
+    ; [TODO]: Needs to be fixed so that this is not duplicated for each test run
+    otx install -v
+allowlist_externals =
+    /bin/bash
 commands =
     coverage erase
-    - coverage run tests/fuzzing/cli_fuzzing.py {posargs:-artifact_prefix={toxworkdir}/ -print_final_stats=1 -atheris_runs=500000}
+    coverage run tests/fuzzing/cli_fuzzing.py {posargs:-artifact_prefix={toxworkdir}/ -jobs=8 -print_final_stats=1 -runs=62500 -dict={toxinidir}/tests/fuzzing/assets/cli/commands.dict}
     coverage report --precision=2
+    /bin/bash -c 'rm {toxinidir}/fuzz-*.log'