FEDOT warm start experiment

.dockerignore

@@ -0,0 +1,13 @@
+# Config & info files
+.pep8speaks.yml
+Dockerfile
+LICENSE
+README.md
+
+# Unnecessary files
+examples
+notebooks
+test
+
+# User data
+data/cache

Dockerfile

@@ -0,0 +1,30 @@
+# Download base image ubuntu 20.04
+FROM ubuntu:20.04
+
+# For apt to be noninteractive
+ENV DEBIAN_FRONTEND noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN true
+
+# Preseed tzdata, update package index, upgrade packages and install needed software
+RUN truncate -s0 /tmp/preseed.cfg; \
+    echo "tzdata tzdata/Areas select Europe" >> /tmp/preseed.cfg; \
+    echo "tzdata tzdata/Zones/Europe select Berlin" >> /tmp/preseed.cfg; \
+    debconf-set-selections /tmp/preseed.cfg && \
+    rm -f /etc/timezone /etc/localtime && \
+	apt-get update && \
+	apt-get install -y nano  && \
+	apt-get install -y mc && \
+    apt-get install -y python3.9 python3-pip && \
+	apt-get install -y git && \
+    rm -rf /var/lib/apt/lists/*
+
+# Set the workdir
+ENV WORKDIR /home/meta-automl-research
+WORKDIR $WORKDIR
+COPY . $WORKDIR
+
+RUN pip3 install pip && \
+    pip install wheel && \
+    pip install --trusted-host pypi.python.org -r ${WORKDIR}/requirements.txt
+
+ENV PYTHONPATH $WORKDIR

README.md

@@ -5,7 +5,7 @@
 [![package](https://badge.fury.io/py/gamlet.svg)](https://badge.fury.io/py/gamlet)
 [![Build](https://github.com/ITMO-NSS-team/MetaFEDOT/actions/workflows/build.yml/badge.svg)](https://github.com/ITMO-NSS-team/MetaFEDOT/actions/workflows/build.yml)
 [![Documentation Status](https://readthedocs.org/projects/gamlet/badge/?version=latest)](https://gamlet.readthedocs.io/en/latest/?badge=latest)
-[![codecov](https://codecov.io/gh/ITMO-NSS-team/GAMLET/graph/badge.svg?token=N3Z9YTPHP9)](https://codecov.io/gh/ITMO-NSS-team/GAMLET)
+<!-- [![codecov](https://codecov.io/gh/ITMO-NSS-team/GAMLET/graph/badge.svg?token=N3Z9YTPHP9)](https://codecov.io/gh/ITMO-NSS-team/GAMLET) -->
 [![Visitors](https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Fgithub.com%2FITMO-NSS-team%2FMetaFEDOT&countColor=%23263759&style=plastic&labelStyle=lower)](https://visitorbadge.io/status?path=https%3A%2F%2Fgithub.com%2FITMO-NSS-team%2FMetaFEDOT)
 
 GAMLET (previously known as MetaFEDOT) is an open platform for sharing meta-learning experiences in **AutoML** and more

experiments/fedot_warm_start/configs/config.yaml

@@ -0,0 +1,20 @@
+---
+seed: 42
+tmpdir: '/var/essdata/tmp'
+update_train_test_datasets_split: true
+
+#data_settings:
+n_datasets: null # null for all available datasets
+test_size: 0.25
+train_timeout: 15
+test_timeout: 15
+n_automl_repetitions: 10
+#meta_learning_params:
+n_best_dataset_models_to_memorize: 10
+mf_extractor_params:
+  groups: general
+assessor_params:
+  n_neighbors: 5
+advisor_params:
+  minimal_distance: 1
+  n_best_to_advise: 3

experiments/fedot_warm_start/configs/config_debug.yaml

@@ -0,0 +1,21 @@
+---
+seed: 42
+save_dir_prefix: debug_
+update_train_test_datasets_split: true
+#data_settings:
+n_datasets: 10 # null for all available datasets
+test_size: 0.4
+train_timeout: 15
+test_timeout: 15
+n_automl_repetitions: 1
+#meta_learning_params:
+n_best_dataset_models_to_memorize: 10
+mf_extractor_params:
+  # groups: general
+  features:
+    - nr_inst
+assessor_params:
+  n_neighbors: 2
+advisor_params:
+  minimal_distance: 1
+  n_best_to_advise: 3

experiments/fedot_warm_start/configs/config_light.yaml

@@ -0,0 +1,19 @@
+---
+seed: 42
+tmpdir: '/var/essdata/tmp'
+save_dir_prefix: light_
+#data_settings:
+n_datasets: 16 # null for all available datasets
+test_size: 0.25
+train_timeout: 15
+test_timeout: 15
+n_automl_repetitions: 10
+#meta_learning_params:
+n_best_dataset_models_to_memorize: 10
+mf_extractor_params:
+  groups: general
+assessor_params:
+  n_neighbors: 5
+advisor_params:
+  minimal_distance: 1
+  n_best_to_advise: 3

experiments/fedot_warm_start/configs/evaluation_config.yaml

@@ -0,0 +1,10 @@
+split_seed: 0
+collect_metrics:
+  - f1
+  - roc_auc
+  - accuracy
+  - neg_log_loss
+  - precision
+baseline_model: 'catboost'
+data_test_size: 0.25
+data_split_seed: 0

experiments/fedot_warm_start/configs/fedot_config.yaml

@@ -0,0 +1,7 @@
+fedot_params:
+  problem: classification
+  logging_level: 10
+  n_jobs: 1
+  show_progress: false
+  cache_dir: '/var/essdata/tmp/fedot_cache'
+  use_auto_preprocessing: true

experiments/fedot_warm_start/configs/use_configs.yaml

@@ -0,0 +1,3 @@
+- config.yaml
+- evaluation_config.yaml
+- fedot_config.yaml

experiments/fedot_warm_start/train_test_datasets_split.csv

@@ -0,0 +1,73 @@
+dataset_id,dataset_name,category,is_train,NumberOfInstances,NumberOfFeatures,NumberOfClasses
+1063,kc2,small_small_binary,1,small,small,binary
+40927,CIFAR_10,big_big_big,1,big,big,big
+1480,ilpd,small_small_binary,1,small,small,binary
+54,vehicle,small_small_small,1,small,small,small
+40978,Internet-Advertisements,big_big_binary,1,big,big,binary
+1464,blood-transfusion-service-center,small_small_binary,1,small,small,binary
+300,isolet,big_big_big,1,big,big,big
+18,mfeat-morphological,small_small_big,1,small,small,big
+23381,dresses-sales,small_small_binary,1,small,small,binary
+46,splice,big_big_small,1,big,big,small
+1461,bank-marketing,big_small_binary,1,big,small,binary
+40966,MiceProtein,small_big_small,1,small,big,small
+40983,wilt,big_small_binary,1,big,small,binary
+469,analcatdata_dmft,small_small_small,1,small,small,small
+1053,jm1,big_small_binary,1,big,small,binary
+40499,texture,big_big_big,1,big,big,big
+40701,churn,big_small_binary,1,big,small,binary
+12,mfeat-factors,small_big_big,1,small,big,big
+1486,nomao,big_big_binary,1,big,big,binary
+40982,steel-plates-fault,small_small_small,1,small,small,small
+1050,pc3,small_big_binary,1,small,big,binary
+307,vowel,small_small_big,1,small,small,big
+1475,first-order-theorem-proving,big_big_small,1,big,big,small
+1049,pc4,small_big_binary,1,small,big,binary
+23517,numerai28.6,big_small_binary,1,big,small,binary
+1468,cnae-9,small_big_big,1,small,big,big
+40984,segment,big_small_small,1,big,small,small
+151,electricity,big_small_binary,1,big,small,binary
+29,credit-approval,small_small_binary,1,small,small,binary
+188,eucalyptus,small_small_small,1,small,small,small
+40668,connect-4,big_big_small,1,big,big,small
+1478,har,big_big_small,1,big,big,small
+22,mfeat-zernike,small_big_big,1,small,big,big
+1067,kc1,small_small_binary,1,small,small,binary
+1487,ozone-level-8hr,big_big_binary,1,big,big,binary
+6332,cylinder-bands,small_big_binary,1,small,big,binary
+1497,wall-robot-navigation,big_small_small,1,big,small,small
+1590,adult,big_small_binary,1,big,small,binary
+16,mfeat-karhunen,small_big_big,1,small,big,big
+1068,pc1,small_small_binary,1,small,small,binary
+3,kr-vs-kp,big_big_binary,1,big,big,binary
+28,optdigits,big_big_big,1,big,big,big
+40996,Fashion-MNIST,big_big_big,1,big,big,big
+1462,banknote-authentication,small_small_binary,1,small,small,binary
+458,analcatdata_authorship,small_big_small,1,small,big,small
+6,letter,big_small_big,1,big,small,big
+40670,dna,big_big_small,1,big,big,small
+1510,wdbc,small_big_binary,1,small,big,binary
+40975,car,small_small_small,1,small,small,small
+4134,Bioresponse,big_big_binary,1,big,big,binary
+37,diabetes,small_small_binary,1,small,small,binary
+44,spambase,big_big_binary,1,big,big,binary
+15,breast-w,small_small_binary,1,small,small,binary
+1501,semeion,small_big_big,1,small,big,big
+40994,climate-model-simulation-crashes,small_small_binary,0,small,small,binary
+4538,GesturePhaseSegmentationProcessed,big_big_small,0,big,big,small
+14,mfeat-fourier,small_big_big,0,small,big,big
+1485,madelon,big_big_binary,0,big,big,binary
+11,balance-scale,small_small_small,0,small,small,small
+23,cmc,small_small_small,0,small,small,small
+554,mnist_784,big_big_big,0,big,big,big
+4534,PhishingWebsites,big_big_binary,0,big,big,binary
+38,sick,big_small_binary,0,big,small,binary
+1494,qsar-biodeg,small_big_binary,0,small,big,binary
+50,tic-tac-toe,small_small_binary,0,small,small,binary
+40979,mfeat-pixel,small_big_big,0,small,big,big
+1489,phoneme,big_small_binary,0,big,small,binary
+31,credit-g,small_small_binary,0,small,small,binary
+32,pendigits,big_small_big,0,big,small,big
+41027,jungle_chess_2pcs_raw_endgame_complete,big_small_small,0,big,small,small
+182,satimage,big_big_small,0,big,big,small
+40923,Devnagari-Script,big_big_big,0,big,big,big

gamlet/approaches/knn_similarity_model_advice.py

@@ -4,6 +4,7 @@
 from typing import Callable, List, Optional, Sequence
 
 from golem.core.optimisers.opt_history_objects.opt_history import OptHistory
+import pandas as pd
 from sklearn.preprocessing import MinMaxScaler
 
 from gamlet.approaches import MetaLearningApproach
@@ -55,7 +56,7 @@ class Components:
     class Data:
         meta_features: DatasetMetaFeatures = None
         datasets: List[OpenMLDataset] = None
-        datasets_data: List[OpenMLDataset] = None
+        datasets_data: List[TabularData] = None
         dataset_ids: List[DatasetIDType] = None
         best_models: List[List[EvaluatedModel]] = None
 
@@ -66,11 +67,11 @@ def fit(self,
         data = self.data
         params = self.parameters
 
-        data.datasets_data = list(datasets_data)
-        data.datasets = [d.dataset for d in datasets_data]
-        data.dataset_ids = [d.id for d in datasets_data]
+        data.meta_features = self.extract_train_meta_features(datasets_data)
+        data.dataset_ids = list(data.meta_features.index)
+        data.datasets_data = [d_d for d_d in datasets_data if d_d.id in data.dataset_ids]
+        data.datasets = [d_d.dataset for d_d in data.datasets_data]
 
-        data.meta_features = self.extract_train_meta_features(data.datasets_data)
         self.fit_datasets_similarity_assessor(data.meta_features, data.dataset_ids)
 
         data.best_models = self.load_models(data.datasets, histories, params.n_best_dataset_models_to_memorize,

gamlet/components/meta_features_extractors/pymfe_extractor.py

@@ -32,8 +32,11 @@ def extract(self, data_sequence: Sequence[Union[DatasetBase, TabularData]],
         for i, dataset_data in enumerate(tqdm(data_sequence, desc='Extracting meta features of the datasets')):
             if isinstance(dataset_data, DatasetBase):
                 dataset_data = dataset_data.get_data()
-            meta_features = self._extract_single(dataset_data, fill_input_nans, fit_kwargs, extract_kwargs)
-            accumulated_meta_features.append(meta_features)
+            try:
+                meta_features = self._extract_single(dataset_data, fill_input_nans, fit_kwargs, extract_kwargs)
+                accumulated_meta_features.append(meta_features)
+            except Exception:
+                logger.exception(f'Dataset {dataset_data.dataset}: error while meta-features extractin.')
 
         output = DatasetMetaFeatures(pd.concat(accumulated_meta_features), is_summarized=self.summarize_features,
                                      features=self.features)

gamlet/data_preparation/datasets_train_test_split.py

@@ -29,24 +29,23 @@ def openml_datasets_train_test_split(dataset_ids: List[OpenMLDatasetIDType], tes
     single_value_categories = cat_counts[cat_counts == 1].index
     idx = df_split_categories[df_split_categories['category'].isin(single_value_categories)].index
     df_split_categories.loc[idx, 'category'] = 'single_value'
-    df_datasets_to_split = df_split_categories[df_split_categories['category'] != 'single_value']
-    df_test_only_datasets = df_split_categories[df_split_categories['category'] == 'single_value']
-    if not df_datasets_to_split.empty:
-        df_train_datasets, df_test_datasets = train_test_split(
-            df_datasets_to_split,
-            test_size=test_size,
-            shuffle=True,
-            stratify=df_datasets_to_split['category'],
-            random_state=seed
-        )
-        df_test_datasets = pd.concat([df_test_datasets, df_test_only_datasets])
+    signle_value_datasets = df_split_categories[df_split_categories['category'] == 'single_value']
+    if len(signle_value_datasets) >= 1:
+        df_datasets_to_split = df_split_categories
+        additional_datasets = pd.DataFrame([])
     else:
-        df_train_datasets, df_test_datasets = train_test_split(
-            df_split_categories,
-            test_size=test_size,
-            shuffle=True,
-            random_state=seed
-        )
+        df_datasets_to_split = df_split_categories[df_split_categories['category'] != 'single_value']
+        additional_datasets = signle_value_datasets
+
+    df_train_datasets, df_test_datasets = train_test_split(
+        df_datasets_to_split,
+        test_size=test_size,
+        shuffle=True,
+        stratify=df_datasets_to_split['category'],
+        random_state=seed
+    )
+    df_train_datasets = pd.concat([df_train_datasets, additional_datasets])
+
     df_train_datasets['is_train'] = 1
     df_test_datasets['is_train'] = 0
     df_split_datasets = pd.concat([df_train_datasets, df_test_datasets]).join(

requirements.txt

@@ -22,3 +22,5 @@ pytest>=7.4.0
 scikit-learn>=1.0.0
 scipy>=1.7.3
 tqdm>=4.65.0
+loguru
+pecapiku @ git+https://github.com/MorrisNein/pecapiku

tests/unit/surrogate/test_surrogate_model.py

@@ -29,10 +29,3 @@ def get_test_data():
     x_pipe = torch.load(path / 'data_pipe_test.pt')
     x_dset = torch.load(path / 'data_dset_test.pt')
     return x_pipe, x_dset
-
-
-def test_model_output(read_config):
-    x_pipe, x_dset = get_test_data()
-    model = create_model_from_config(read_config, x_pipe, x_dset)
-    pred = torch.squeeze(model.forward(x_pipe, x_dset))
-    assert pred.shape[0] == 256