[MCBO]: refactor to prepare the suppoort of multi-obj / constrained opt.

MCBO/README.md

@@ -131,7 +131,7 @@ if __name__ == '__main__':
         x = optimizer.suggest(1)
         y = task(x)
         optimizer.observe(x, y)
-        print(f'Iteration {i + 1:3d}/{100:3d} - f(x) = {y[0, 0]:.3f} - f(x*) = {optimizer.best_y:.3f}')
+        print(f'Iteration {i + 1:3d}/{100:3d} - f(x) = {y[0, 0]:.3f} - f(x*) = {optimizer.best_y.item():.3f}')
 
     # Access history of suggested points and black-box values
     all_x = search_space.inverse_transform(optimizer.data_buffer.x)
@@ -186,6 +186,7 @@ in [general_plot_utils.py](./mcbo/utils/general_plot_utils.py).
 
 ## Library Roadmap
 
+#### Features
 - [x] Allows restart from checkpoints
 - [x] Add random tree-based additive GP kernel as surrogate model
 - [x] Add message-passing acquisition function optimizer
@@ -198,6 +199,12 @@ in [general_plot_utils.py](./mcbo/utils/general_plot_utils.py).
 - [ ] Handle black-box constraints.
 - [ ] Handle multi-fidelity MCBO.
 - [ ] Implement probabilistic reparameterization for acquisition function optimization
+- [ ] Support optimizing in a table of points instead of in the full search space.
+
+#### Debug
+- [] improve the way we cope with hallucinatory points (to prevent sampled values from exploding)
+
+
 
 We invite you to contribute to the development of the MCBO library notably by proposing implementation of new modules,
 providing new tasks or spotting issues.

MCBO/experiments/all_runs.sh

@@ -15,7 +15,7 @@ for task in ackley aig_optimization antibody_design mig_optimization pest rna_in
       done
 
       acq_func="ts"
-      model=lr_sparse
+      model=lr_sparse_hs
       opt_id="${model}__${acq_opt}__${acq_func}__${tr}"
       cmd="python ./experiments/run_task_exps.py --device_id 0 --absolut_dir $ABSOLUT_EXE --task_id $task --optimizers_ids $opt_id --seeds $SEEDS"
       $cmd
@@ -43,4 +43,9 @@ for task in ackley-53 xgboost_opt aig_optimization_hyp svm_opt; do
       done
     done
   done
+
+  for opt_id in ga sa rs ls; do
+    cmd="python ./experiments/run_task_exps.py --device_id 0 --task_id $task --optimizers_ids $opt_id --seeds $SEEDS"
+    $cmd
+  done
 done

MCBO/experiments/all_runs_test.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+MAX_NUM_ITER=30
+BO_N_INIT=4
+RESULT_DIR="./test_results/"
+VERBOSE=2
+# --- Combinatorial exps ---
+
+SEEDS="0"
+ABSOLUT_EXE="./libs/Absolut/src/AbsolutNoLib"
+for task in ackley aig_optimization antibody_design mig_optimization pest rna_inverse_fold; do
+  for acq_opt in ls is sa ga; do
+    for tr in "basic" "none"; do
+      acq_func="ei"
+      for model in gp_o gp_to gp_ssk gp_diff gp_hed; do
+        opt_id="${model}__${acq_opt}__${acq_func}__${tr}"
+        cmd="python ./experiments/run_task_exps.py --device_id 0 --absolut_dir $ABSOLUT_EXE --task_id $task --optimizers_ids $opt_id --seeds $SEEDS --max_num_iter $MAX_NUM_ITER --bo_n_init $BO_N_INIT --result_dir $RESULT_DIR --verbose $VERBOSE"
+        echo $cmd
+        $cmd
+      done
+
+      acq_func="ts"
+      model=lr_sparse_hs
+      opt_id="${model}__${acq_opt}__${acq_func}__${tr}"
+      cmd="python ./experiments/run_task_exps.py --device_id 0 --absolut_dir $ABSOLUT_EXE --task_id $task --optimizers_ids $opt_id --seeds $SEEDS --max_num_iter $MAX_NUM_ITER --bo_n_init $BO_N_INIT --result_dir $RESULT_DIR --verbose $VERBOSE"
+      echo $cmd
+      $cmd
+    done
+  done
+
+  for opt_id in ga sa rs ls; do
+    cmd="python ./experiments/run_task_exps.py --device_id 0 --absolut_dir $ABSOLUT_EXE --task_id $task --optimizers_ids $opt_id --seeds $SEEDS --max_num_iter $MAX_NUM_ITER --bo_n_init $BO_N_INIT --result_dir $RESULT_DIR --verbose $VERBOSE"
+    echo $cmd
+    $cmd
+  done
+done
+
+# --- Mixed exps ---
+
+SEEDS="0"
+for task in ackley-53 xgboost_opt aig_optimization_hyp svm_opt; do
+  for acq_opt in mab is sa ga; do
+    for tr in "basic" "none"; do
+      acq_func="ei"
+      for model in gp_o gp_to gp_hed; do
+        opt_id="${model}__${acq_opt}__${acq_func}__${tr}"
+        cmd="python ./experiments/run_task_exps.py --device_id 0 --task_id $task --optimizers_ids $opt_id --seeds $SEEDS --max_num_iter $MAX_NUM_ITER --bo_n_init $BO_N_INIT --result_dir $RESULT_DIR --verbose $VERBOSE"
+        echo $cmd
+        $cmd
+      done
+    done
+  done
+
+  for opt_id in ga sa rs ls; do
+    cmd="python ./experiments/run_task_exps.py --device_id 0 --task_id $task --optimizers_ids $opt_id --seeds $SEEDS --max_num_iter $MAX_NUM_ITER --bo_n_init $BO_N_INIT --result_dir $RESULT_DIR --verbose $VERBOSE"
+    echo $cmd
+    $cmd
+  done
+done

MCBO/experiments/run_task_exps.py

@@ -7,50 +7,56 @@
 
 sys.path.insert(0, str(Path(os.path.realpath(__file__)).parent.parent))
 
-
 if __name__ == '__main__':
     from mcbo.utils.experiment_utils import run_experiment, get_task_from_id, get_opt
+
     parser = argparse.ArgumentParser(add_help=True, description='MCBO')
     parser.add_argument("--device_id", type=int, default=0, help="Cuda device id (cpu is used if id is negative)")
-    parser.add_argument("--task_id", type=str, required=True, help="Name of the task")
+    parser.add_argument("--task_id", type=str, nargs="+", required=True, help="Name of the task")
     parser.add_argument("--optimizers_ids", type=str, nargs="+", required=True, help="Name of the methods to run")
     parser.add_argument("--seeds", type=int, nargs="+", required=True, help="Seeds to run")
     parser.add_argument("--verbose", type=int, default=1, help="Verbosity level")
+    parser.add_argument("--result_dir", type=str, default=None, help="Root of the result dir (./results by default)")
+    parser.add_argument("--max_num_iter", type=int, default=200, help="Number of acquisitions")
+    parser.add_argument("--bo_n_init", type=int, default=20,
+                        help="Number of points to acquire before running acquisition with BO")
 
     # Antigen binding task
     parser.add_argument("--absolut_dir", type=str, default=None, required=False, help="Path to Absolut! executer.")
 
     args = parser.parse_args()
 
     dtype_ = torch.float64
-    task_id_ = args.task_id
-    task = get_task_from_id(task_id=task_id_, absolut_dir=args.absolut_dir)
-    search_space = task.get_search_space(dtype=dtype_)
-
-    bo_n_init_ = 20
-    if args.device_id >= 0 and torch.cuda.is_available():
-        bo_device_ = torch.device(f'cuda:{args.device_id}')
-    else:
-        bo_device_ = torch.device("cpu")
-
-    max_num_iter = 200
-    random_seeds = args.seeds
-
-    selected_optimizers = []
-    for opt_id in args.optimizers_ids:
-        opt = get_opt(
-            task=task,
-            short_opt_id=opt_id,
-            bo_n_init=bo_n_init_,
-            dtype=dtype_,
-            bo_device=bo_device_
-        )
-
-        run_experiment(
-            task=task,
-            optimizers=[opt],
-            random_seeds=random_seeds,
-            max_num_iter=max_num_iter,
-            save_results_every=max_num_iter,
-            very_verbose=args.verbose > 1
-        )
+    task_ids_ = args.task_id
+    for task_id_ in task_ids_:
+        task = get_task_from_id(task_id=task_id_, absolut_dir=args.absolut_dir)
+        search_space = task.get_search_space(dtype=dtype_)
+
+        bo_n_init_ = args.bo_n_init
+        if args.device_id >= 0 and torch.cuda.is_available():
+            bo_device_ = torch.device(f'cuda:{args.device_id}')
+        else:
+            bo_device_ = torch.device("cpu")
+
+        max_num_iter = args.max_num_iter
+        random_seeds = args.seeds
+
+        selected_optimizers = []
+        for opt_id in args.optimizers_ids:
+            opt = get_opt(
+                task=task,
+                short_opt_id=opt_id,
+                bo_n_init=bo_n_init_,
+                dtype=dtype_,
+                bo_device=bo_device_
+            )
+
+            run_experiment(
+                task=task,
+                optimizers=[opt],
+                random_seeds=random_seeds,
+                max_num_iter=max_num_iter,
+                save_results_every=max_num_iter,
+                very_verbose=args.verbose > 1,
+                result_dir=args.result_dir
+            )

MCBO/mcbo/acq_funcs/acq_base.py

@@ -56,6 +56,10 @@ def evaluate(self,
         """
         pass
 
+    @abstractmethod
+    def __call__(self, x: torch.Tensor, model: Union[ModelBase, EnsembleModelBase], **kwargs) -> torch.Tensor:
+        pass
+
 
 class ConstrAcqBase(AcqBase, ABC):
 
@@ -107,9 +111,13 @@ def __call__(self, x: torch.Tensor, model: Union[ModelBase, EnsembleModelBase],
             acq = acq_values.mean(dim=1)
 
         else:
-            acq = self.evaluate(x=x.to(device, dtype), model=model,
-                                constr_models=constr_models, out_upper_constr_vals=out_upper_constr_vals,
-                                **kwargs)
+            acq = self.evaluate(
+                x=x.to(device, dtype),
+                model=model,
+                constr_models=constr_models,
+                out_upper_constr_vals=out_upper_constr_vals,
+                **kwargs
+            )
 
         if ndim == 1:
             acq = acq.squeeze(0)

MCBO/mcbo/acq_funcs/additive_lcb.py

@@ -23,9 +23,11 @@ def evaluate(self,
                  model: RandDecompositionGP,
                  **kwargs
                  ) -> torch.Tensor:
-        val = torch.tensor(0)
+        val = torch.tensor([0])
         for clique in model.graph:
-            val += self.partial_evaluate(x, model, clique, **kwargs)
+            aux = self.partial_evaluate(x, model, clique, **kwargs)
+            val = val.to(aux)
+            val += aux
 
         return val
 

MCBO/mcbo/acq_funcs/cei.py

@@ -6,7 +6,7 @@
 # This program is distributed in the hope that it will be useful, but WITHOUT ANY
 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
 # PARTICULAR PURPOSE. See the MIT License for more details.
-from typing import Union, List
+from typing import Union, List, Optional
 
 import torch
 from torch.distributions import Normal
@@ -44,7 +44,7 @@ def evaluate(self,
                  model: ModelBase,
                  constr_models: List[ModelBase],
                  out_upper_constr_vals: torch.Tensor,
-                 best_y: Union[float, torch.Tensor],
+                 best_y: Optional[Union[float, torch.Tensor]],
                  **kwargs
                  ) -> torch.Tensor:
         """
@@ -53,9 +53,13 @@ def evaluate(self,
         Args:
             out_upper_constr_vals: upper bound for constraints
             constr_models: model for each output associated to a constraint
+            best_y: best observed objective value so far, if None then optimize feasibility probability
         """
         # Get `- EI(x)`
-        neg_ei = self.ei.evaluate(x=x, model=model, best_y=best_y, **kwargs)
+        if best_y is None:
+            neg_ei = -1
+        else:
+            neg_ei = self.ei.evaluate(x=x, model=model, best_y=best_y, **kwargs)
 
         # Get Pr(c_i(x) <= lambda_i)
         if isinstance(constr_models, ModelBase):

MCBO/mcbo/acq_optimizers/acq_optimizer_base.py

@@ -41,6 +41,9 @@ def __init__(self,
                  search_space: SearchSpace,
                  dtype: torch.dtype,
                  input_constraints: Optional[List[Callable[[Dict], bool]]],
+                 obj_dims: Union[List[int], np.ndarray, None],
+                 out_constr_dims: Union[List[int], np.ndarray, None],
+                 out_upper_constr_vals: Optional[torch.Tensor],
                  **kwargs
                  ):
         """
@@ -49,11 +52,26 @@ def __init__(self,
             dtype: tensor type
             input_constraints: list of funcs taking a point as input and outputting whether the point
                                        is valid or not
+            obj_dims: dimensions in ys corresponding to objective values to minimize
+            out_constr_dims: dimensions in ys corresponding to inequality constraints
+            out_upper_constr_vals: values of upper bounds for inequality constraints
+
         """
         self.search_space = search_space
         self.dtype = dtype
         self.input_constraints = input_constraints
         self.kwargs = kwargs
+        if obj_dims is None:
+            obj_dims = np.array([0])
+        if out_constr_dims is None:
+            out_constr_dims = []
+            out_upper_constr_vals = []
+        self.obj_dims = obj_dims
+        self.out_constr_dims = out_constr_dims
+        self.out_upper_constr_vals = out_upper_constr_vals
+        assert len(self.obj_dims) == 1, "Do not support multi-obj for now"
+        assert len(self.out_constr_dims) == 0, "Do not support constraints for now"
+        assert len(self.out_upper_constr_vals) == 0, "Do not support constraints for now"
 
     @abstractmethod
     def optimize(self,
@@ -87,7 +105,7 @@ def optimize(self,
         """
         pass
 
-    def post_observe_method(self, x: torch.Tensor, y: torch.Tensor, 
+    def post_observe_method(self, x: torch.Tensor, y: torch.Tensor,
                             data_buffer: DataBuffer, n_init: int, **kwargs) -> None:
         """
         Function called at the end of observe method. Can be used to update the internal state of the acquisition
@@ -112,7 +130,7 @@ def input_eval_from_transfx(self, transf_x: torch.Tensor) -> np.ndarray:
                 specifying at index `(i, j)` if input point `i` is valid regarding constraint function `j`        
         """
         return input_eval_from_transfx(transf_x=transf_x, search_space=self.search_space,
-                                               input_constraints=self.input_constraints)
+                                       input_constraints=self.input_constraints)
 
     def input_eval_from_origx(self, x: Union[pd.DataFrame, Dict]) -> np.ndarray:
         """
@@ -128,7 +146,7 @@ def input_eval_from_origx(self, x: Union[pd.DataFrame, Dict]) -> np.ndarray:
         return input_eval_from_origx(x=x, input_constraints=self.input_constraints)
 
     def sample_input_valid_points(self, n_points: int, point_sampler: Callable[[int], pd.DataFrame],
-                                          max_trials: int = 100) -> pd.DataFrame:
+                                  max_trials: int = 100) -> pd.DataFrame:
         """ Get valid points in original space
 
         Args: