Framework design templates

scripts/select_random_files_eval.py

@@ -0,0 +1,40 @@
+import random
+from pathlib import Path
+
+
+def select_files_randomly(population, sample_size):
+    if sample_size > len(population):
+        print("Sample size is larger than the population size.")
+        return []
+
+    selected_files = random.sample(population, sample_size)
+
+    return selected_files
+
+
+analysis_sensitivities = "../micro-benchmark/analysis_sensitivities"
+python_features = "../micro-benchmark/python_features"
+file_extension = "*.py"
+
+
+analysis_sensitivities_population = sorted(Path(analysis_sensitivities).rglob("*.py"))
+python_features_population = sorted(Path(python_features).rglob("*.py"))
+
+
+sample_size = 10  # Desired sample size
+selected_files_analysis_sensitivities = select_files_randomly(
+    analysis_sensitivities_population, sample_size
+)
+selected_files_python_features = select_files_randomly(
+    python_features_population, sample_size
+)
+
+print("Selected files for evaluation from analysis_sensitivities:")
+for file_name in selected_files_analysis_sensitivities:
+    print(file_name)
+
+print("\n####\n")
+
+print("Selected files for evaluation from python_features:")
+for file_name in selected_files_python_features:
+    print(file_name)

src/tests_runner/tests_runner.py → src/main_runner.py

@@ -1,31 +1,39 @@
+import logging
 import os
 import tarfile
 import time
+from datetime import datetime
 from io import BytesIO
 
 import docker
 
+# Create a logger
+logger = logging.getLogger("Main Runner")
+logger.setLevel(logging.DEBUG)
 
-class FileHandler:
-    def _copy_files_to_container(self, container, src, dst):
-        original_directory = os.getcwd()
-        src_dirname = os.path.dirname(src)
-        src_basename = os.path.basename(src)
+file_handler = logging.FileHandler("main_runner.log")
+file_handler.setLevel(logging.DEBUG)
 
-        if os.path.isfile(src):
-            os.chdir(src_dirname)
-        else:
-            os.chdir(os.path.dirname(src))
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
 
-        tar = tarfile.open(src_basename + ".tar", mode="w")
-        try:
-            tar.add(src_basename)
-        finally:
-            tar.close()
 
-        data = open(src + ".tar", "rb").read()
-        container.put_archive(os.path.dirname(dst), data)
-        os.chdir(original_directory)
+class FileHandler:
+    def _copy_files_to_container(self, container, src, dst):
+        # Create tar of micro-bench folder
+        temp_path = "/tmp/temp.tar"
+        with tarfile.open(temp_path, "w:gz") as tar:
+            base_folder = os.path.basename(src)
+            tar.add(src, arcname=base_folder)
+
+        with open(temp_path, "rb") as file:
+            data = file.read()
+            container.put_archive(dst, data)
 
     def _copy_files_from_container(self, container, src, dst):
         stream, _ = container.get_archive(src)
@@ -53,91 +61,103 @@ def __init__(self, tool_name, dockerfile_path):
         self.tool_name = tool_name
         self.dockerfile_path = dockerfile_path
         self.dockerfile_name = tool_name
-        self.script_path = f"./{self.tool_name}_script.py"
-        self.docker_script_path = f"/tmp/{self.tool_name}_script.py"
+        self.test_runner_script_path = f"/tmp/src/runner.py"
+        self.host_results_path = f"./results_{datetime.now().strftime('%d-%m %H:%M')}"
+
+        if not os.path.exists(self.host_results_path):
+            os.makedirs(self.host_results_path)
 
     def _build_docker_image(self):
-        print("Building image")
+        logger.info("Building image")
         image, _ = self.docker_client.images.build(
             path=self.dockerfile_path, tag=self.dockerfile_name
         )
         return image
 
     def _spawn_docker_instance(self):
-        print("Creating container")
+        logger.info("Creating container")
         container = self.docker_client.containers.run(
-            self.dockerfile_name, detach=True, stdin_open=True, tty=True
+            self.dockerfile_name,
+            detach=True,
+            stdin_open=True,
+            tty=True,
         )
         return container
 
     def _run_test_in_session(self, result_path="/tmp/micro-benchmark"):
-        print("#####################################################")
-        print("Running :", self.tool_name)
+        logger.info("#####################################################")
+        logger.info(f"Running : {self.tool_name}")
         self._build_docker_image()
         container = self._spawn_docker_instance()
+
         file_handler = FileHandler()
-        src = "../../micro-benchmark"
-        dst = "/tmp/micro-benchmark"
+        src = "../micro-benchmark"
+        dst = "/tmp"
         file_handler._copy_files_to_container(container, src, dst)
-        file_handler._copy_files_to_container(
-            container, self.script_path, self.docker_script_path
-        )
-        print("Type inferring.. ")
+
+        logger.info("Type inferring...")
         start_time = time.time()
-        container.exec_run(f"python {self.docker_script_path}")
+        _, response = container.exec_run(
+            f"python {self.test_runner_script_path}", stream=True
+        )
+        for line in response:
+            logger.info(line)
+
         end_time = time.time()
         execution_time = end_time - start_time
-        print(f"Execution time: {execution_time} seconds")
+        logger.info(f"Execution time: {execution_time} seconds")
+
         file_handler._copy_files_from_container(
-            container, result_path, self.dockerfile_path
+            container, result_path, f"{self.host_results_path}/{self.tool_name}"
         )
+
         container.stop()
         container.remove()
 
 
 class ScalpelRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("scalpel", "../target_tools/scalpel")
+        super().__init__("scalpel", "./target_tools/scalpel")
 
     def run_tool_test(self):
         self._run_test_in_session()
 
 
 class PyreRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("pyre", "../target_tools/pyre")
+        super().__init__("pyre", "./target_tools/pyre")
 
     def run_tool_test(self):
         self._run_test_in_session()
 
 
 class PyrightRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("pyright", "../target_tools/pyright")
+        super().__init__("pyright", "./target_tools/pyright")
 
     def run_tool_test(self):
         self._run_test_in_session("/tmp/typings")
 
 
 class PytypeRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("pytype", "../target_tools/pytype")
+        super().__init__("pytype", "./target_tools/pytype")
 
     def run_tool_test(self):
         self._run_test_in_session()
 
 
 class JediRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("jedi", "../target_tools/jedi")
+        super().__init__("jedi", "./target_tools/jedi")
 
     def run_tool_test(self):
         self._run_test_in_session()
 
 
 class HityperRunner(TypeEvalPyRunner):
     def __init__(self):
-        super().__init__("hityper", "../target_tools/hityper")
+        super().__init__("hityper", "./target_tools/hityper")
 
     def run_tool_test(self):
         self._run_test_in_session()

src/target_tools/scalpel/Dockerfile

@@ -17,5 +17,7 @@ COPY requirements.txt /app/requirements.txt
 RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 
+COPY src /tmp/src
+
 # Run the application
 CMD ["bash"]

src/target_tools/scalpel/src/runner.py

@@ -0,0 +1,63 @@
+import argparse
+import json
+import logging
+import os
+from pathlib import Path
+
+from scalpel.typeinfer.typeinfer import TypeInference
+
+# Create a logger
+logger = logging.getLogger("runner")
+logger.setLevel(logging.DEBUG)
+
+file_handler = logging.FileHandler("/tmp/scalpel_log.log")
+file_handler.setLevel(logging.DEBUG)
+
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
+
+
+def list_python_files(folder_path):
+    python_files = sorted(Path(folder_path).rglob("*.py"))
+    return python_files
+
+
+def main_runner(args):
+    python_files = list_python_files(args.bechmark_path)
+    error_count = 0
+    for file in python_files:
+        try:
+            logger.info(file)
+            inferer = TypeInference(name=file, entry_point=file)
+            inferer.infer_types()
+            inferred = inferer.get_types()
+            json_file_path = str(file).replace(".py", "_result.json")
+
+            with open(json_file_path, "w") as json_file:
+                inferred_serializable = [
+                    {k: list(v) if isinstance(v, set) else v for k, v in d.items()}
+                    for d in inferred
+                ]
+                json.dump(inferred_serializable, json_file, indent=4)
+        except Exception as e:
+            logger.info(f"Command returned non-zero exit status: {e} for file: {file}")
+            error_count += 1
+
+    logger.info(f"Runner finished with errors:{error_count}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--bechmark_path",
+        help="Specify the benchmark path",
+        default="/tmp/micro-benchmark",
+    )
+
+    args = parser.parse_args()
+    main_runner(args)

src/target_tools/template/src/runner.py

@@ -1,7 +1,22 @@
 import argparse
-import json
-import os
+import logging
 from pathlib import Path
+from sys import stdout
+
+# Create a logger
+logger = logging.getLogger("runner")
+logger.setLevel(logging.DEBUG)
+
+file_handler = logging.FileHandler("/tmp/<tool_name>_log.log")
+file_handler.setLevel(logging.DEBUG)
+
+console_handler = logging.StreamHandler(stdout)
+console_handler.setLevel(logging.DEBUG)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+file_handler.setFormatter(formatter)
+console_handler.setFormatter(formatter)
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)
 
 
 def list_python_files(folder_path):
@@ -18,10 +33,10 @@ def main_runner(args):
             pass
 
         except Exception as e:
-            print(f"Command returned non-zero exit status: {e} for file: {file}")
+            logger.info(f"Command returned non-zero exit status: {e} for file: {file}")
             error_count += 1
 
-    print(f"Runner finished with errors:{error_count}")
+    logger.info(f"Runner finished with errors:{error_count}")
 
 
 if __name__ == "__main__":