Changed data type of trials argument. Added main documentation file.

README.md

@@ -0,0 +1,29 @@
+<div align="center">
+<h2>Configuration-free imbalanced learning</h2>
+
+The long-term goal of the project is to facilitate automated design of machine learning pipelines in the case of imbalanced distribution of target classes.
+</div>
+
+### Project status
+Currently, only binary classification setting is implemented.
+<br/>
+<br/>
+Benchmark experiments are available for [Auto-gluon](https://github.com/autogluon/autogluon) and Imba.
+### Prerequisites
+
+1. Python interpreter 3.10.
+2. Installation of requirements.
+
+### Usage example
+
+To run a benchmark just type in the terminal:
+```
+./experiment.sh
+```
+
+By default, benchmark for Imba will be run. To change to **Auto-gluon**, add the `ag` argument.
+<br/>
+<br/>
+Stdout is in a file. To change to **console** output, add the `c` argument.
+
+

experiment.sh

@@ -7,23 +7,20 @@ run_experiment() {
   declare out='file'
   declare automl='imba'
   declare preset='good_quality'
-  declare trials=30
+  declare trials=0
 
   # TODO: check for flags for comparison.
-  if [[ "$*" == *"c"* ]]
-  then
+  if [[ "$*" == *"c"* ]]; then
     out="console"
   fi
 
-  if [[ "$*" == *"c"* ]]
-    then
-      out="console"
-    fi
+  if [[ "$*" == *"ag"* ]]; then
+    automl="ag"
+  fi
 
-  if [[ "$*" == *"50"* ]]
-      then
-        trials=50
-      fi
+  if [[ "$*" == *"50"* ]]; then
+    trials=50
+  fi
 
 #    if [[ ${args[2]} == "c" ]]; then
 #        out="console"

experiment/main.py

@@ -4,11 +4,13 @@
 import shutil
 import sys
 from datetime import datetime
+from typing import Optional
 
 from setuptools import setup
 
 import numpy as np
 from experiment.runner import OpenMLExperimentRunner
+from pathlib import Path
 
 
 class ExperimentMain:
@@ -18,19 +20,24 @@ def main():
         parser.add_argument('--automl', action='store', dest='automl', default='imba')
         parser.add_argument('--out', action='store', dest='out', default='file')
         parser.add_argument('--preset', action='store', dest='preset', default='good_quality')
-        parser.add_argument('--trials', action='store', dest='trials', default=30)
+        parser.add_argument('--trials', action='store', dest='trials', type=int, default=None)
 
         args = parser.parse_args()
         automl = getattr(args, 'automl')
         logging_output = getattr(args, 'out')
         autogluon_preset = getattr(args, 'preset')
         trials = getattr(args, 'trials')
 
+        if trials is not None and trials == 0:
+            trials = None
+
         if logging_output == 'file':
             if automl == 'ag':
                 log_file_name = 'logs/AG/'
             else:
                 log_file_name = 'logs/Imba/'
+
+            Path(log_file_name).mkdir(parents=True, exist_ok=True)
             log_file_name += datetime.now().strftime('%Y-%m-%d %H:%M') + '.log'
 
             logging.basicConfig(
@@ -46,11 +53,11 @@ def main():
                 format='%(asctime)s - %(levelname)s - %(message)s'
             )
         else:
-            raise Exception("Invalid --o option. Options available: ['file', 'console'].")
+            raise ValueError("Invalid --out option. Options available: ['file', 'console'].")
 
         if automl == 'ag':
             if autogluon_preset not in ['medium_quality', 'good_quality']:
-                raise Exception("Invalid --preset option. Options available: ['medium_quality', 'good_quality'].")
+                raise ValueError("Invalid --preset option. Options available: ['medium_quality', 'good_quality'].")
 
             from experiment.autogluon import AGExperimentRunner
 
@@ -60,7 +67,7 @@ def main():
 
             runner = ImbaExperimentRunner()
         else:
-            raise Exception("Invalid --automl option. Options available: ['imba', 'ag'].")
+            raise ValueError("Invalid --automl option. Options available: ['imba', 'ag'].")
 
         runner.define_tasks()
 

experiment/runner.py

@@ -31,7 +31,7 @@ class ExperimentRunner(ABC):
     def __init__(self, *args, **kwargs):
         self._tasks: List[Dataset, ...] = []
         self._id_counter = itertools.count(start=1)
-        self._n_evals = 10
+        self._n_evals = 30
         self._fitted_model: FittedModel
 
         self._configure_environment()

requirements.txt

@@ -0,0 +1,6 @@
+hyperopt==0.2.7
+imbalanced-ensemble==0.2.2
+imbalanced-learn==0.12.3
+openml==0.14.2
+ray==2.40.0
+scikit-learn==1.5.1