add hpobench wrapper

.github/workflows/build.yml

@@ -98,6 +98,10 @@ jobs:
       run: |
           python -m pip install --upgrade pip
           pip install tox tox-gh-actions
+    - name: Install Hpobench
+      uses: eWaterCycle/setup-singularity@v7
+      with:
+        singularity-version: 3.8.7
     - name: Test with tox (and all extra dependencies)
       run: tox -e py-all
     - name: Upload coverage to Codecov

docs/src/code/benchmark/task.rst

@@ -16,3 +16,4 @@ Task modules
    task/rosenbrock
    task/forrester
    task/profet
+   task/hpobench

docs/src/code/benchmark/task/hpobench.rst

@@ -0,0 +1,5 @@
+HPOBench
+=============================
+
+.. automodule:: orion.benchmark.task.hpobench
+   :members:

docs/src/user/benchmark.rst

@@ -27,6 +27,17 @@ Beside out of box :doc:`/code/benchmark/task` and :doc:`/code/benchmark/assessme
 you can also extend benchmark to add new ``Tasks`` with :doc:`/code/benchmark/task/base` and
 ``Assessments`` with :doc:`/code/benchmark/assessment/base`,
 
+To run benchmark with task :doc:`/code/benchmark/task/hpobench`, use ``pip install orion[hpobench]``
+to install the extra requirements. HPOBench provides local and containerized benchmarks, but different
+local benchmark could ask for total difference extra requirements. With ``pip install orion[hpobench]``,
+you will be able to run local benchmark ``benchmarks.ml.tabular_benchmark``.
+If you want to run other benchmarks in local, refer HPOBench `Run a Benchmark Locally`_. To run
+containerized benchmarks, you will need to install `singularity`_. You can choose to run local
+or containerized benchmarks by providing different `hpo_benchmark_class` parameter for
+:doc:`/code/benchmark/task/hpobench`.
+
 Learn how to get start using benchmark in Orion with this `sample notebook`_.
 
+.. _Run a Benchmark Locally: https://github.com/automl/HPOBench#run-a-benchmark-locally
+.. _singularity: https://singularity.hpcng.org/admin-docs/master/installation.html
 .. _sample notebook: https://github.com/Epistimio/orion/tree/develop/examples/benchmark/benchmark_get_start.ipynb

setup.py

@@ -63,6 +63,10 @@
         "pymoo==0.5.0",
         "hebo @ git+https://github.com/huawei-noah/[email protected]#egg=hebo&subdirectory=HEBO",
     ],
+    "hpobench": [
+        "openml",
+        "hpobench @ git+https://github.com/automl/HPOBench.git@master#egg=hpobench",
+    ],
 }
 extras_require["all"] = sorted(set(sum(extras_require.values(), [])))
 

src/orion/algo/space/configspace.py

@@ -23,6 +23,7 @@
         IntegerHyperparameter,
         NormalFloatHyperparameter,
         NormalIntegerHyperparameter,
+        OrdinalHyperparameter,
         UniformFloatHyperparameter,
         UniformIntegerHyperparameter,
     )
@@ -44,6 +45,7 @@ class DummyType:
     UniformIntegerHyperparameter = DummyType
     NormalIntegerHyperparameter = DummyType
     CategoricalHyperparameter = DummyType
+    OrdinalHyperparameter = DummyType
 
 
 class UnsupportedPrior(Exception):
@@ -185,6 +187,13 @@ def _from_categorical(dim: CategoricalHyperparameter) -> Categorical:
     return Categorical(dim.name, choices)
 
 
+@to_oriondim.register(OrdinalHyperparameter)
+def _from_ordinal(dim: OrdinalHyperparameter) -> Categorical:
+    """Builds a categorical dimension from a categorical hyperparameter"""
+    choices = list(dim.sequence)
+    return Categorical(dim.name, choices)
+
+
 @to_oriondim.register(UniformIntegerHyperparameter)
 @to_oriondim.register(UniformFloatHyperparameter)
 def _from_uniform(dim: Hyperparameter) -> Integer | Real:

src/orion/benchmark/task/__init__.py

@@ -8,6 +8,7 @@
 from .carromtable import CarromTable
 from .eggholder import EggHolder
 from .forrester import Forrester
+from .hpobench import HPOBench
 from .rosenbrock import RosenBrock
 
 try:
@@ -25,6 +26,7 @@
     "EggHolder",
     "Forrester",
     "profet",
+    "HPOBench",
     # "ProfetSvmTask",
     # "ProfetFcNetTask",
     # "ProfetForresterTask",

src/orion/benchmark/task/hpobench.py

@@ -0,0 +1,95 @@
+"""
+Task for HPOBench
+=================
+"""
+import importlib
+import subprocess
+from typing import Dict, List
+
+from orion.algo.space.configspace import to_orionspace
+from orion.benchmark.task.base import BenchmarkTask
+from orion.core.utils.module_import import ImportOptional
+
+with ImportOptional("HPOBench", "hpobench") as import_optional:
+    from hpobench import __version__ as hpobench_version
+
+    print(f"HPOBench version: {hpobench_version}")
+
+
+class HPOBench(BenchmarkTask):
+    """Benchmark Task wrapper over HPOBench (https://github.com/automl/HPOBench)
+
+    For more information on HPOBench, see original paper at https://arxiv.org/abs/2109.06716.
+
+    Katharina Eggensperger, Philipp Müller, Neeratyoy Mallik, Matthias Feurer, René Sass, Aaron Klein,
+    Noor Awad, Marius Lindauer, Frank Hutter. "HPOBench: A Collection of Reproducible Multi-Fidelity
+    Benchmark Problems for HPO" Thirty-fifth Conference on Neural Information Processing Systems
+    Datasets and Benchmarks Track (Round 2).
+
+    Parameters
+    ----------
+    max_trials : int
+        Maximum number of trials for this task.
+    hpo_benchmark_class : str
+        Full path to a particular class of benchmark in HPOBench.
+    benchmark_kwargs: str
+        Optional parameters to create benchmark instance of class `hpo_benchmark_class`.
+    objective_function_kwargs: dict
+        Optional parameters to use when calling `objective_function` of the benchmark instance.
+    """
+
+    def __init__(
+        self,
+        max_trials: int,
+        hpo_benchmark_class: str = None,
+        benchmark_kwargs: dict = None,
+        objective_function_kwargs: dict = None,
+    ):
+        import_optional.ensure()
+        super().__init__(
+            max_trials=max_trials,
+            hpo_benchmark_class=hpo_benchmark_class,
+            benchmark_kwargs=benchmark_kwargs,
+            objective_function_kwargs=objective_function_kwargs,
+        )
+        self._verify_benchmark(hpo_benchmark_class)
+        self.hpo_benchmark_cls = self._load_benchmark(hpo_benchmark_class)
+        self.benchmark_kwargs = dict() if benchmark_kwargs is None else benchmark_kwargs
+        self.objective_function_kwargs = (
+            dict() if objective_function_kwargs is None else objective_function_kwargs
+        )
+
+    def call(self, **kwargs) -> List[Dict]:
+        hpo_benchmark = self.hpo_benchmark_cls(**self.benchmark_kwargs)
+        result_dict = hpo_benchmark.objective_function(
+            configuration=kwargs, **self.objective_function_kwargs
+        )
+        objective = result_dict["function_value"]
+        return [
+            dict(
+                name=self.hpo_benchmark_cls.__name__, type="objective", value=objective
+            )
+        ]
+
+    def _load_benchmark(self, hpo_benchmark_class: str):
+        package, cls = hpo_benchmark_class.rsplit(".", 1)
+        module = importlib.import_module(package)
+        return getattr(module, cls)
+
+    def _verify_benchmark(self, hpo_benchmark_class: str):
+        if not hpo_benchmark_class:
+            raise AttributeError("Please provide full path to a HPOBench benchmark")
+        if "container" in hpo_benchmark_class:
+            code, message = subprocess.getstatusoutput("singularity -h")
+            if code != 0:
+                raise AttributeError(
+                    "Can not run containerized benchmark without Singularity: {}".format(
+                        message
+                    )
+                )
+
+    def get_search_space(self) -> Dict[str, str]:
+        configuration_space = self.hpo_benchmark_cls(
+            **self.benchmark_kwargs
+        ).get_configuration_space()
+        return to_orionspace(configuration_space)

tests/unittests/algo/test_configspace.py

@@ -52,10 +52,10 @@ def test_orion_configspace():
 
 def test_configspace_to_orion_unsupported():
     from ConfigSpace import ConfigurationSpace
-    from ConfigSpace.hyperparameters import OrdinalHyperparameter
+    from ConfigSpace.hyperparameters import Constant
 
     cspace = ConfigurationSpace()
-    cspace.add_hyperparameters([OrdinalHyperparameter("a", (1, 2, 0, 3))])
+    cspace.add_hyperparameters([Constant("a", 100)])
 
     with pytest.raises(NotImplementedError):
         _ = to_orionspace(cspace)

tests/unittests/benchmark/task/test_hpobench.py

@@ -0,0 +1,170 @@
+import inspect
+
+import pytest
+
+from orion.algo.space import Space
+from orion.benchmark.task import HPOBench
+from orion.benchmark.task.hpobench import import_optional
+
+hpobench_benchmarks = list()
+hpobench_benchmarks.append(
+    {
+        "type": "tabular",
+        "class": "hpobench.container.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+        "init_args": dict(model="xgb", task_id=168912),
+        "objective_args": dict(),
+        "hyperparams": {
+            "colsample_bytree": 1.0,
+            "eta": 0.045929204672575,
+            "max_depth": 1,
+            "reg_lambda": 10.079368591308594,
+        },
+    }
+)
+
+hpobench_benchmarks.append(
+    {
+        "type": "raw",
+        "class": "hpobench.container.benchmarks.ml.xgboost_benchmark.XGBoostBenchmark",
+        "init_args": dict(task_id=168912),
+        "objective_args": dict(),
+        "hyperparams": {
+            "colsample_bytree": 1.0,
+            "eta": 0.045929204672575,
+            "max_depth": 1,
+            "reg_lambda": 10.079368591308594,
+        },
+    }
+)
+
+"""
+# need fix of https://github.com/Epistimio/orion/issues/1018
+hpobench_benchmarks.append({
+    "type": "surrogate",
+    "class": "hpobench.container.benchmarks.surrogates.paramnet_benchmark.ParamNetAdultOnStepsBenchmark",
+    "init_args": dict(),
+    "objective_args": dict(),
+    "hyperparams": {
+        "average_units_per_layer_log2": 6.0,
+        "batch_size_log2": 5.5,
+        "dropout_0": 0.25,
+        "dropout_1": 0.25,
+        "final_lr_fraction_log2": 1.0
+    }
+})
+"""
+
+
+@pytest.mark.skipif(
+    import_optional.failed,
+    reason="Running without HPOBench",
+)
+class TestHPOBench:
+    """Test benchmark task HPOBenchWrapper"""
+
+    def test_create_with_non_container_benchmark(self):
+        """Test to create HPOBench local benchmark"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        assert task.max_trials == 2
+        assert inspect.isclass(task.hpo_benchmark_cls)
+        assert task.configuration == {
+            "HPOBench": {
+                "hpo_benchmark_class": "hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+                "benchmark_kwargs": {"model": "xgb", "task_id": 168912},
+                "objective_function_kwargs": None,
+                "max_trials": 2,
+            }
+        }
+
+    def test_create_with_container_benchmark(self):
+        """Test to create HPOBench container benchmark"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.container.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        assert task.max_trials == 2
+        assert inspect.isclass(task.hpo_benchmark_cls)
+        assert task.configuration == {
+            "HPOBench": {
+                "hpo_benchmark_class": "hpobench.container.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+                "benchmark_kwargs": {"model": "xgb", "task_id": 168912},
+                "objective_function_kwargs": None,
+                "max_trials": 2,
+            }
+        }
+
+    def test_run_locally(self):
+        """Test to run a local HPOBench benchmark"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        params = {
+            "colsample_bytree": 1.0,
+            "eta": 0.045929204672575,
+            "max_depth": 1.0,
+            "reg_lambda": 10.079368591308594,
+        }
+
+        objectives = task(**params)
+        assert objectives == [
+            {
+                "name": "TabularBenchmark",
+                "type": "objective",
+                "value": 0.056373193166885674,
+            }
+        ]
+
+    @pytest.mark.parametrize("benchmark", hpobench_benchmarks)
+    def test_run_singulariys(self, benchmark):
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class=benchmark.get("class"),
+            benchmark_kwargs=benchmark.get("init_args"),
+            objective_function_kwargs=benchmark.get("objective_args"),
+        )
+
+        params = benchmark.get("hyperparams")
+        objectives = task(**params)
+
+        assert len(objectives) > 0
+
+    def test_run_singulariy(self):
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.container.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        params = {
+            "colsample_bytree": 1.0,
+            "eta": 0.045929204672575,
+            "max_depth": 1.0,
+            "reg_lambda": 10.079368591308594,
+        }
+
+        objectives = task(**params)
+        assert objectives == [
+            {
+                "name": "TabularBenchmark",
+                "type": "objective",
+                "value": 0.056373193166885674,
+            }
+        ]
+
+    def test_search_space(self):
+        """Test to get task search space"""
+        task = HPOBench(
+            max_trials=2,
+            hpo_benchmark_class="hpobench.benchmarks.ml.tabular_benchmark.TabularBenchmark",
+            benchmark_kwargs=dict(model="xgb", task_id=168912),
+        )
+        space = task.get_search_space()
+
+        assert isinstance(space, Space)
+        assert len(space) == 4

tests/unittests/benchmark/task/test_tasks.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """Tests for :mod:`orion.benchmark.task`."""
 
+
 from orion.benchmark.task import Branin, CarromTable, EggHolder, RosenBrock
 
 

tests/unittests/client/test_runner.py

@@ -631,6 +631,13 @@ def test_runner_inside_childprocess():
     else:
         # parent process wait for child process to end
         wpid, exit_status = os.wait()
+        if wpid != pid:
+            try:
+                while wpid != pid:
+                    wpid, exit_status = os.wait()
+            except:
+                pass
+
         assert wpid == pid
         assert exit_status == 0