add ConditionedSpace to support complex conditions between hyperparam…

…eters (#37)

openbox/acq_maximizer/ei_optimization.py

@@ -8,10 +8,10 @@
 import abc
 import logging
 import time
+import warnings
 from typing import Iterable, List, Union, Tuple, Optional
-
 import random
-import scipy
+import scipy.optimize
 import numpy as np
 
 from openbox.acquisition_function.acquisition import AbstractAcquisitionFunction
@@ -360,6 +360,8 @@ def _one_iter(
             if (not changed_inc) or \
                     (self.max_steps is not None and
                      local_search_steps == self.max_steps):
+                if len(time_n) == 0:
+                    time_n.append(0.0)
                 self.logger.debug("Local search took %d steps and looked at %d "
                                   "configurations. Computing the acquisition "
                                   "value for one configuration took %f seconds"
@@ -599,24 +601,31 @@ def maximize(
         def negative_acquisition(x):
             # shape of x = (d,)
             x = np.clip(x, 0.0, 1.0)    # fix numerical problem in L-BFGS-B
+            try:
+                # self.config_space._check_forbidden(x)
+                Configuration(self.config_space, vector=x).is_valid_configuration()
+            except ValueError:
+                return np.inf
             return -self.acquisition_function(x, convert=False)[0]  # shape=(1,)
 
         if initial_config is None:
             initial_config = self.config_space.sample_configuration()
         init_point = initial_config.get_array()
 
         acq_configs = []
-        result = scipy.optimize.minimize(fun=negative_acquisition,
-                                         x0=init_point,
-                                         bounds=self.bounds,
-                                         **self.scipy_config)
-        # if result.success:
-        #     acq_configs.append((result.fun, Configuration(self.config_space, vector=result.x)))
+        with warnings.catch_warnings():
+            # ignore warnings of np.inf
+            warnings.filterwarnings("ignore", message="invalid value encountered in subtract", category=RuntimeWarning)
+            result = scipy.optimize.minimize(fun=negative_acquisition,
+                                             x0=init_point,
+                                             bounds=self.bounds,
+                                             **self.scipy_config)
         if not result.success:
             self.logger.debug('Scipy optimizer failed. Info:\n%s' % (result,))
         try:
             x = np.clip(result.x, 0.0, 1.0)  # fix numerical problem in L-BFGS-B
             config = Configuration(self.config_space, vector=x)
+            config.is_valid_configuration()
             acq = self.acquisition_function(x, convert=False)
             acq_configs.append((acq, config))
         except Exception:

openbox/utils/space.py

@@ -2,10 +2,12 @@
 
 # More readable wrapped classes based on ConfigSpace.
 # The comments are modified based on https://github.com/automl/ConfigSpace/blob/master/ConfigSpace/hyperparameters.pyx
-from typing import List, Dict, Tuple, Union, Optional
+from typing import List, Dict, Tuple, Union, Optional, Callable
+import numpy as np
 import ConfigSpace as CS
 from ConfigSpace import EqualsCondition, InCondition, ForbiddenAndConjunction, ForbiddenEqualsClause, ForbiddenInClause
 from ConfigSpace import ConfigurationSpace, Configuration
+from ConfigSpace.exceptions import ForbiddenValueError
 
 
 # Hyperparameters
@@ -200,6 +202,7 @@ def __init__(
             name: Union[str, None] = None,
             seed: Union[int, None] = None,
             meta: Optional[Dict] = None,
+            **kwargs,
     ) -> None:
         """
         A collection-like object containing a set of variable definitions and conditions.
@@ -220,10 +223,118 @@ def __init__(
             Field for holding meta data provided by the user.
             Not used by the configuration space.
         """
-        super().__init__(name=name, seed=seed, meta=meta)
+        super().__init__(name=name, seed=seed, meta=meta, **kwargs)
 
     def add_variables(self, variables: List[Variable]):
         self.add_hyperparameters(variables)
 
     def add_variable(self, variable: Variable):
         self.add_hyperparameter(variable)
+
+
+class ConditionedSpace(Space):
+    """
+    A configuration space that supports complex conditions between hyperparameters/variables,
+        e.g., x1 <= x2 and x1 * x2 < 100.
+
+    User can define a sample_condition function to restrict the generation of configurations.
+
+    The following functions are guaranteed to return valid configurations:
+        - self.sample_configuration()
+        - get_one_exchange_neighbourhood()  # may return empty list
+
+    Example
+    -------
+
+    >>> def sample_condition(config):
+    >>>     # require x1 <= x2 and x1 * x2 < 100
+    >>>     if config['x1'] > config['x2']:
+    >>>         return False
+    >>>     if config['x1'] * config['x2'] >= 100:
+    >>>         return False
+    >>>     return True
+    >>>
+    >>> cs = ConditionedSpace()
+    >>> cs.add_variables([...])
+    >>> cs.set_sample_condition(sample_condition)  # set the sample condition after all variables are added
+    >>> configs = cs.sample_configuration(1000)
+
+    """
+    sample_condition: Callable[[Configuration], bool] = None
+
+    def set_sample_condition(self, sample_condition: Callable[[Configuration], bool]):
+        """
+        The sample_condition function takes a configuration as input and returns a boolean value.
+            - If the return value is True, the configuration is valid and will be sampled.
+            - If the return value is False, the configuration is invalid and will be rejected.
+        This function should be called after all hyperparameters/variables are added to the conditioned space.
+        """
+        self.sample_condition = sample_condition
+        self._check_default_configuration()
+
+    def _check_forbidden(self, vector: np.ndarray) -> None:
+        """
+        This function is called in Configuration.is_valid_configuration().
+            - When Configuration.__init__() is called with values (dict), is_valid_configuration() is called.
+            - When Configuration.__init__() is called with vectors (np.ndarray), there will be no validation check.
+        This function is also called in get_one_exchange_neighbourhood().
+        """
+        # check original forbidden clauses first
+        super()._check_forbidden(vector)
+
+        if self.sample_condition is not None:
+            # Populating a configuration from an array does not check if it is a legal configuration.
+            # _check_forbidden() is not called. Otherwise, this would be stuck in an infinite loop.
+            config = Configuration(self, vector=vector)
+            if not self.sample_condition(config):
+                raise ForbiddenValueError('User-defined sample condition is not satisfied.')
+
+    def sample_configuration(self, size: int = 1) -> Union['Configuration', List['Configuration']]:
+        """
+        In ConfigurationSpace.sample_configuration, configurations are built with vectors (np.ndarray),
+            so there will be no validation check and _check_forbidden() will not be called.
+            We need to check the sample condition manually.
+
+        Returns a single configuration if size = 1 else a list of Configurations
+        """
+        if self.sample_condition is None:
+            return super().sample_configuration(size=size)
+
+        if not isinstance(size, int):
+            raise TypeError('Argument size must be of type int, but is %s'
+                            % type(size))
+        elif size < 1:
+            return []
+
+        error_iteration = 0
+        accepted_configurations = []  # type: List['Configuration']
+        while len(accepted_configurations) < size:
+            missing = size - len(accepted_configurations)
+
+            if missing != size:
+                missing = int(1.1 * missing)
+            missing += 2
+
+            configurations = super().sample_configuration(size=missing)  # missing > 1, return a list
+            configurations = [c for c in configurations if self.sample_condition(c)]
+            if len(configurations) > 0:
+                accepted_configurations.extend(configurations)
+            else:
+                error_iteration += 1
+                if error_iteration > 1000:
+                    raise ForbiddenValueError("Cannot sample valid configuration for %s" % self)
+
+        if size <= 1:
+            return accepted_configurations[0]
+        else:
+            return accepted_configurations[:size]
+
+    def add_hyperparameter(self, *args, **kwargs):
+        if self.sample_condition is not None:
+            raise ValueError('Please add hyperparameter/variable before setting sample condition.')
+        return super().add_hyperparameter(*args, **kwargs)
+
+    def add_hyperparameters(self, *args, **kwargs):
+        if self.sample_condition is not None:
+            raise ValueError('Please add hyperparameters/variables before setting sample condition.')
+        return super().add_hyperparameters(*args, **kwargs)

test/space/test_conditioned_space.py

@@ -0,0 +1,74 @@
+import sys
+sys.path.insert(0, '.')
+
+import traceback
+from openbox import sp
+from openbox.utils.config_space import get_one_exchange_neighbourhood
+from ConfigSpace import Configuration
+import numpy as np
+import pickle as pkl
+
+
+verbose = True
+
+
+def sample_condition(config):
+    if verbose:
+        print('- sample_condition called')
+    # We want x1 >= x2 >= x3
+    if config['x1'] < config['x2']:
+        return False  # violate condition
+    if config['x2'] < config['x3']:
+        return False  # violate condition
+    return True  # valid
+
+
+if __name__ == '__main__':
+    cs = sp.ConditionedSpace()
+    for i in range(1, 4):
+        cs.add_hyperparameter(sp.Int('x%d' % i, 0, 100, default_value=0))
+    print(cs)
+    cs.set_sample_condition(sample_condition)
+    cs.seed(1234)
+    print('space initialized.')
+
+    print('\nsample 10 configs:')
+    configs = cs.sample_configuration(10)
+    print('\nvalidate configs:')
+    for config in configs:
+        config.is_valid_configuration()
+        is_valid = sample_condition(config)
+        print(config, is_valid)
+        if not is_valid:
+            raise ValueError('sampled invalid config')
+
+    print('\ntest invalid values (dict)')
+    try:
+        config = Configuration(cs, values=dict(x1=1, x2=2, x3=3))
+    except ValueError as e:
+        traceback.print_exc()
+        print('invalid values caught')
+    print('\ntest invalid vector (np.array)')
+    config = Configuration(cs, vector=np.array([0.1, 0.2, 0.3]))  # will not check in initialization with vector
+    try:
+        config.is_valid_configuration()
+    except ValueError as e:
+        traceback.print_exc()
+        print('invalid vector caught')
+    print('\ntest invalid neighbors')
+    config = Configuration(cs, values=dict(x1=100, x2=50, x3=10))
+    neighbors = list(get_one_exchange_neighbourhood(config, seed=2))
+    print(len(neighbors), 'neighbors')
+    assert all([sample_condition(config) for config in neighbors])
+
+    verbose = False
+    print('\nsample 10000 configs')
+    configs = cs.sample_configuration(10000)
+    assert all([sample_condition(config) for config in configs])
+
+    print('\ntest pickle')
+    bcs = pkl.dumps(cs)
+    cs = pkl.loads(bcs)
+    cs.sample_configuration(10)
+
+    print('\nAll tests passed!')

test/space/test_conditioned_space_branin.py

@@ -0,0 +1,50 @@
+import sys
+sys.path.insert(0, '.')
+
+from openbox import sp, Optimizer
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def sample_condition(config):
+    # We want x1 >= x2
+    if config['x1'] < config['x2']:
+        return False  # violate condition
+    return True  # valid
+
+
+# Define Search Space
+space = sp.ConditionedSpace()
+x1 = sp.Real("x1", -5, 10, default_value=0)
+x2 = sp.Real("x2", 0, 15, default_value=0)
+space.add_variables([x1, x2])
+space.set_sample_condition(sample_condition)
+
+
+# Define Objective Function
+def branin(config):
+    x1, x2 = config['x1'], config['x2']
+    y = (x2 - 5.1 / (4 * np.pi ** 2) * x1 ** 2 + 5 / np.pi * x1 - 6) ** 2 \
+        + 10 * (1 - 1 / (8 * np.pi)) * np.cos(x1) + 10
+    return {'objs': (y,)}
+
+
+# Run
+if __name__ == "__main__":
+    opt = Optimizer(
+        branin,
+        space,
+        max_runs=50,
+        surrogate_type='gp',
+        acq_optimizer_type='random_scipy',
+        time_limit_per_trial=30,
+        task_id='test_cs',
+        random_state=1,
+    )
+    history = opt.run()
+
+    print(history)
+
+    history.plot_convergence(true_minimum=0.397887)
+    plt.show()
+