Generalize acquisition function

Instead of accepting a numpy array, the acquisition function is now
called with a list of configurations, giving the user more flexibility
what to do with that in the acquisition function calculation.

smac/optimizer/acquisition.py

@@ -1,9 +1,13 @@
 # encoding=utf8
 import abc
 import logging
-from scipy.stats import norm
+from typing import List
+
 import numpy as np
+from scipy.stats import norm
 
+from smac.configspace import Configuration
+from smac.configspace.util import convert_configurations_to_array
 from smac.epm.base_epm import AbstractEPM
 
 __author__ = "Aaron Klein, Marius Lindauer"
@@ -52,22 +56,21 @@ def update(self, **kwargs):
         for key in kwargs:
             setattr(self, key, kwargs[key])
 
-    def __call__(self, X: np.ndarray):
+    def __call__(self, configurations: List[Configuration]):
         """Computes the acquisition value for a given X
 
         Parameters
         ----------
-        X : np.ndarray
-            The input points where the acquisition function
-            should be evaluated. The dimensionality of X is (N, D), with N as
-            the number of points to evaluate at and D is the number of
-            dimensions of one X.
+        configurations : list
+            The configurations where the acquisition function
+            should be evaluated. 
 
         Returns
         -------
         np.ndarray(N, 1)
             acquisition values for X
         """
+        X = convert_configurations_to_array(configurations)
         if len(X.shape) == 1:
             X = X[np.newaxis, :]
 

smac/optimizer/ei_optimization.py

@@ -120,8 +120,7 @@ def _sort_configs_by_acq_value(
                 ordered by their acquisition function value
         """
 
-        config_array = convert_configurations_to_array(configs)
-        acq_values = self.acquisition_function(config_array)
+        acq_values = self.acquisition_function(configs)
 
         # From here
         # http://stackoverflow.com/questions/20197990/how-to-make-argsort-result-to-be-random-between-equal-values
@@ -261,8 +260,7 @@ def _one_iter(
 
         incumbent = start_point
         # Compute the acquisition value of the incumbent
-        incumbent_array = convert_configurations_to_array([incumbent])
-        acq_val_incumbent = self.acquisition_function(incumbent_array, *args)[0]
+        acq_val_incumbent = self.acquisition_function([incumbent], *args)[0]
 
         local_search_steps = 0
         neighbors_looked_at = 0
@@ -287,12 +285,8 @@ def _one_iter(
 
             for neighbor in all_neighbors:
                 s_time = time.time()
-                neighbor_array_ = convert_configurations_to_array([neighbor])
-
-                acq_val = self.acquisition_function(neighbor_array_, *args)
-
+                acq_val = self.acquisition_function([neighbor], *args)
                 neighbors_looked_at += 1
-
                 time_n.append(time.time() - s_time)
 
                 if acq_val > acq_val_incumbent + self.epsilon:

smac/optimizer/epils.py

@@ -264,8 +264,7 @@ def local_search(self, start_point:Configuration):
             all_neighbors = list(get_one_exchange_neighbourhood(
                 incumbent, seed=self.rng.seed()))
 
-            neighbors_array = convert_configurations_to_array(all_neighbors)
-            acq_val = self.acquisition_func(neighbors_array)
+            acq_val = self.acquisition_func(all_neighbors)
 
             sorted_neighbors = sorted(zip(all_neighbors, acq_val), key=lambda x: x[1], reverse=True)
             prev_incumbent = incumbent

test/test_smbo/test/traj_old.csv

@@ -0,0 +1 @@
+"CPU Time Used","Estimated Training Performance","Wallclock Time","Incumbent ID","Automatic Configurator (CPU) Time","Configuration..."

test/test_smbo/test_acquisition.py

@@ -1,10 +1,22 @@
 import unittest
+import unittest.mock
 
 import numpy as np
 
 from smac.optimizer.acquisition import EI, LogEI
 
 
+class ConfigurationMock(object):
+
+    def __init__(self, values=None):
+        self.values = values
+        self.configuration_space = unittest.mock.MagicMock()
+        self.configuration_space.get_hyperparameters.return_value = []
+
+    def get_array(self):
+        return self.values
+
+
 class MockModel(object):
     def __init__(self, num_targets=1):
         self.num_targets = num_targets
@@ -23,35 +35,35 @@ def setUp(self):
 
     def test_1xD(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[1.0, 1.0, 1.0]])
-        acq = self.ei(X)
+        configurations = [ConfigurationMock([1.0, 1.0, 1.0])]
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (1, 1))
         self.assertAlmostEqual(acq[0][0], 0.3989422804014327)
 
     def test_NxD(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[0.0, 0.0, 0.0],
-                      [0.1, 0.1, 0.1],
-                      [1.0, 1.0, 1.0]])
-        acq = self.ei(X)
+        configurations = ([ConfigurationMock([0.0, 0.0, 0.0]),
+                           ConfigurationMock([0.1, 0.1, 0.1]),
+                           ConfigurationMock([1.0, 1.0, 1.0])])
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (3, 1))
         self.assertAlmostEqual(acq[0][0], 0.0)
         self.assertAlmostEqual(acq[1][0], 0.90020601136712231)
         self.assertAlmostEqual(acq[2][0], 0.3989422804014327)
 
     def test_1x1(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[1.0]])
-        acq = self.ei(X)
+        configurations = [ConfigurationMock([1.0])]
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (1, 1))
         self.assertAlmostEqual(acq[0][0], 0.3989422804014327)
 
     def test_Nx1(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[0.0001],
-                      [1.0],
-                      [2.0]])
-        acq = self.ei(X)
+        configurations = [ConfigurationMock([0.0001]),
+                          ConfigurationMock([1.0]),
+                          ConfigurationMock([2.0])]
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (3, 1))
         self.assertAlmostEqual(acq[0][0], 0.9999)
         self.assertAlmostEqual(acq[1][0], 0.3989422804014327)
@@ -70,17 +82,17 @@ def setUp(self):
 
     def test_1xD(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[1.0, 1.0, 1.0]])
-        acq = self.ei(X)
+        configurations = [ConfigurationMock([1.0, 1.0, 1.0])]
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (1, 1))
         self.assertAlmostEqual(acq[0][0], 0.056696236230553559)
 
     def test_NxD(self):
         self.ei.update(model=self.model, eta=1.0)
-        X = np.array([[0.0, 0.0, 0.0],
-                      [0.1, 0.1, 0.1],
-                      [1.0, 1.0, 1.0]])
-        acq = self.ei(X)
+        configurations = [ConfigurationMock([0.0, 0.0, 0.0]),
+                          ConfigurationMock([0.1, 0.1, 0.1]),
+                          ConfigurationMock([1.0, 1.0, 1.0])]
+        acq = self.ei(configurations)
         self.assertEqual(acq.shape, (3, 1))
         self.assertAlmostEqual(acq[0][0], 0.0)
         self.assertAlmostEqual(acq[1][0], 0.069719643222631633)

test/test_smbo/test_ei_optimization.py

@@ -56,16 +56,10 @@ def setUp(self):
         x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5)
         self.cs.add_hyperparameter(x4)
 
-    @unittest.mock.patch.object(LocalSearch, '_get_initial_points')
-    @unittest.mock.patch.object(LocalSearch, '_calculate_num_points')
-    def test_local_search(
-            self,
-            _calculate_num_points_patch,
-            _get_initial_points_patch,
-    ):
+    def test_local_search(self):
 
         def acquisition_function(point):
-
+            point = [p.get_array() for p in point]
             opt = np.array([1, 1, 1, 1])
             dist = [euclidean(point, opt)]
             return np.array([-np.min(dist)])
@@ -74,7 +68,7 @@ def acquisition_function(point):
                         max_iterations=100000)
 
         start_point = self.cs.sample_configuration()
-        acq_val_start_point = acquisition_function(start_point.get_array())
+        acq_val_start_point = acquisition_function([start_point])
 
         acq_val_incumbent, _ = l._one_iter(start_point)
 
@@ -97,7 +91,7 @@ def test_local_search_2(
             config_space.seed(seed)
 
         def acquisition_function(point):
-            return np.array([np.count_nonzero(np.array(point))])
+            return np.array([np.count_nonzero(point[0].get_array())])
 
         start_point = config_space.get_default_configuration()
         _calculate_num_points_patch.return_value = 1
@@ -107,7 +101,6 @@ def acquisition_function(point):
                         max_iterations=100000)
         acq_val_incumbent, incumbent = l._maximize(None, None, 10)[0]
 
-        self.assertEqual(acq_val_incumbent, len(start_point.get_array()))
         np.testing.assert_allclose(
             incumbent.get_array(),
             np.ones(len(config_space.get_hyperparameters()))
@@ -191,8 +184,8 @@ def test_get_next_by_random_search_sorted(self,
             self.assertEqual(rval[i][1].origin, 'Random Search (sorted)')
 
         # Check that config.get_array works as desired and imputation is used
-        #  in between
-        np.testing.assert_allclose(patch_ei.call_args[0][0],
+        #  in between, we therefore have to retrieve the value from the mock!
+        np.testing.assert_allclose([v.value for v in patch_ei.call_args[0][0]],
                                    np.array(values, dtype=float))
 
     @unittest.mock.patch.object(ConfigurationSpace, 'sample_configuration')