add wandb sweep configs

config/hydra/benchmark_optimizer.yaml

@@ -1,11 +1,12 @@
 defaults:
+  - _self_
   - optimizer: discrete_evolution
   - test_function: ehrlich
 
 random_seed: 0
 log_interval: 1
 num_opt_steps: 512
-wandb_host: null
+wandb_host: https://api.wandb.ai
 wandb_mode: online
 project_name: holo
 exp_name: dry_run

config/wandb/optimizer_hparam_sweep.yaml

@@ -0,0 +1,50 @@
+project: holo
+entity: samuelstanton
+method: bayes
+metric:
+  goal: minimize
+  name: cumulative_regret
+
+parameters:
+  exp_name:
+    value: discrete_evolution_hyper_sweep
+  random_seed:
+    value: 0
+
+  # Test Function Parameters
+  test_function.dim:
+    value: 256
+  test_function.num_motifs:
+    value: 4
+  test_function.motif_length:
+    value: 8
+  test_function.quantization:
+    value: 4
+  test_function.noise_std:
+    value: 0.0
+
+  # Optimizer Parameters
+  optimizer.mutation_prob:
+    # values: [0.02, 0.0039]
+    distribution: log_normal
+    mu: -4.5
+    sigma: 1
+  optimizer.recombine_prob:
+    # values: [0.0882, 0.0039]
+    distribution: log_normal
+    mu: -4.5
+    sigma: 1
+  optimizer.survival_quantile:
+    # values: [0.00015, 0.0001]
+    distribution: log_normal
+    mu: -8
+    sigma: 1
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

config/wandb/reproduce_fig_4a.yaml

@@ -0,0 +1,30 @@
+project: holo
+entity: samuelstanton
+method: grid
+
+parameters:
+  exp_name:
+    value: ehrlich_param_sweep
+  random_seed:
+    distribution: int_uniform
+    min: 1
+    max: 32
+
+  # Test Function Parameters
+  test_function.dim:
+    values: [128, 256, 512]
+  test_function.num_motifs:
+    value: 4
+  test_function.motif_length:
+    value: 4
+  test_function.noise_std:
+    value: 0.0
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

config/wandb/reproduce_fig_4b.yaml

@@ -0,0 +1,30 @@
+project: holo
+entity: samuelstanton
+method: grid
+
+parameters:
+  exp_name:
+    value: ehrlich_param_sweep
+  random_seed:
+    distribution: int_uniform
+    min: 1
+    max: 32
+
+  # Test Function Parameters
+  test_function.dim:
+    value: 256
+  test_function.num_motifs:
+    values: [4, 8, 16]
+  test_function.motif_length:
+    value: 4
+  test_function.noise_std:
+    value: 0.0
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

config/wandb/reproduce_fig_4c.yaml

@@ -0,0 +1,30 @@
+project: holo
+entity: samuelstanton
+method: grid
+
+parameters:
+  exp_name:
+    value: ehrlich_param_sweep
+  random_seed:
+    distribution: int_uniform
+    min: 1
+    max: 32
+
+  # Test Function Parameters
+  test_function.dim:
+    value: 256
+  test_function.num_motifs:
+    value: 4
+  test_function.motif_length:
+    values: [4, 8, 10]
+  test_function.noise_std:
+    value: 0.0
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

config/wandb/reproduce_fig_4d.yaml

@@ -0,0 +1,32 @@
+project: holo
+entity: samuelstanton
+method: grid
+
+parameters:
+  exp_name:
+    value: ehrlich_param_sweep
+  random_seed:
+    distribution: int_uniform
+    min: 1
+    max: 32
+
+  # Test Function Parameters
+  test_function.dim:
+    value: 256
+  test_function.num_motifs:
+    value: 4
+  test_function.motif_length:
+    value: 8
+  test_function.quantization:
+    values: [2, 4, 8]
+  test_function.noise_std:
+    value: 0.0
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

config/wandb/reproduce_fig_5.yaml

@@ -0,0 +1,41 @@
+project: holo
+entity: samuelstanton
+method: grid
+
+parameters:
+  exp_name:
+    value: discrete_evolution_hyper_comparison
+  random_seed:
+    distribution: int_uniform
+    min: 1
+    max: 32
+
+  # Test Function Parameters
+  test_function.dim:
+    value: 256
+  test_function.num_motifs:
+    value: 4
+  test_function.motif_length:
+    value: 8
+  test_function.quantization:
+    value: 4
+  test_function.noise_std:
+    value: 0.0
+
+  # Optimizer Parameters
+  optimizer.mutation_prob:
+    values: [0.0144, 0.0039]
+  optimizer.recombine_prob:
+    values: [0.0084, 0.0039]
+  optimizer.survival_quantile:
+    values: [0.0002, 0.0001]
+
+
+program: scripts/benchmark_optimizer.py
+command:
+  - ${env}
+  - ${interpreter}
+  - ${program}
+  - "--config-name"
+  - "benchmark_optimizer"
+  - ${args_no_hyphens}

scripts/benchmark_optimizer.py

@@ -3,13 +3,13 @@
 import hydra
 import numpy as np
 import torch
-import wandb
 from omegaconf import OmegaConf
 
+import wandb
 from holo.logging import wandb_setup
 
 
-@hydra.main(config_path="../config/hydra", config_name="benchmark_optimizer")
+@hydra.main(version_base=None, config_path="../config/hydra", config_name="benchmark_optimizer")
 def main(cfg):
     if cfg.optimizer.mutation_prob is None:
         cfg.optimizer.mutation_prob = 1.1 / cfg.test_function.dim