diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6ace4b3f..0572482a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -40,5 +40,5 @@ jobs: run: poetry install - name: Run pytest - timeout-minutes: 10 + timeout-minutes: 15 run: poetry run pytest -m "all_examples or metahyper" diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py b/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py index 8c9af9e6..f677f894 100644 --- a/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py +++ b/src/neps/optimizers/bayesian_optimization/acquisition_functions/mf_ei.py @@ -32,30 +32,6 @@ def __init__( def get_budget_level(self, config) -> int: return int((config.fidelity.value - config.fidelity.lower) / self.b_step) - # def _preprocess_tabular(self, x: pd.Series) -> pd.Series: - # if len(x) == 0: - # return x - # # extract fid name - # _x = x.loc[0].hp_values() - # _x.pop("id") - # fid_name = list(_x.keys())[0] - # for i in x.index.values: - # # extracting actual HPs from the tabular space - # _config = self.pipeline_space.custom_grid_table.loc[x.loc[i]["id"].value].to_dict() - # # updating fidelities as per the candidate set passed - # _config.update({fid_name: x.loc[i][fid_name].value}) - # # placeholder config from the raw tabular space - # config = self.pipeline_space.raw_tabular_space.sample( - # patience=100, - # user_priors=True, - # ignore_fidelity=True # True allows fidelity to appear in the sample - # ) - # # copying values from table to placeholder config of type SearchSpace - # config.load_from(_config) - # # replacing the ID in the candidate set with the actual HPs of the config - # x.loc[i] = config - # return x - def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]: """Prepares the configurations for appropriate EI calculation. @@ -68,7 +44,6 @@ def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]: # preprocess tabular space differently # expected input: IDs pertaining to the tabular data # expected output: IDs pertaining to current observations and set of HPs - # x = self._preprocess_tabular(x) x = map_real_hyperparameters_from_tabular_ids(x, self.pipeline_space) indices_to_drop = [] for i, config in x.items(): @@ -77,18 +52,17 @@ def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]: # IMPORTANT to set the fidelity at which EI will be calculated only for # the partial configs that have been observed already target_fidelity = config.fidelity.value + self.b_step - config.fidelity.value = min( - target_fidelity, config.fidelity.upper - ) # to respect the bounded fidelity + + if np.less_equal(target_fidelity, config.fidelity.upper): + # only consider the configs with fidelity lower than the max fidelity + config.fidelity.value = target_fidelity + budget_list.append(self.get_budget_level(config)) + else: + # if the target_fidelity higher than the max drop the configuration + indices_to_drop.append(i) else: config.fidelity.value = target_fidelity - - if np.isclose(target_fidelity, config.fidelity.value): - # the fidelity was set the configuration will be considered budget_list.append(self.get_budget_level(config)) - else: - # the fidelity was not set, the configuration will be dropped - indices_to_drop.append(i) # Drop unused configs x.drop(labels=indices_to_drop, inplace=True) @@ -103,22 +77,19 @@ def preprocess(self, x: pd.Series) -> Tuple[Iterable, Iterable]: inc_list.append(inc) return x, torch.Tensor(inc_list) - + def preprocess_gp(self, x: Iterable) -> Tuple[Iterable, Iterable]: x, inc_list = self.preprocess(x) return x.values.tolist(), inc_list - + def preprocess_deep_gp(self, x: Iterable) -> Tuple[Iterable, Iterable]: x, inc_list = self.preprocess(x) x_lcs = [] for idx in x.index: if idx in self.observations.df.index.levels[0]: - budget_level = max(0, self.get_budget_level(x[idx]) - 1) - lc = self.observations.extract_learning_curve( - idx, budget_level - ) + budget_level = self.get_budget_level(x[idx]) + lc = self.observations.extract_learning_curve(idx, budget_level) else: - # TODO: comment to explain why this is needed (karibbov) # initialize a learning curve with a place holder # This is later padded accordingly for the Conv1D layer lc = [0.0] @@ -137,26 +108,32 @@ def preprocess_pfn(self, x: Iterable) -> Tuple[Iterable, Iterable, Iterable]: len_partial = len(self.observations.seen_config_ids) z_min = x[0].fidelity.lower # converting fidelity to the discrete budget level - # STRICT ASSUMPTION: fidelity is the second dimension - _x_tok[:len_partial, 1] = (_x_tok[:len_partial, 1] + self.b_step - z_min) / self.b_step + # STRICT ASSUMPTION: fidelity is the first dimension + _x_tok[:len_partial, 0] = ( + _x_tok[:len_partial, 0] + self.b_step - z_min + ) / self.b_step return _x_tok, _x, inc_list - def eval( - self, x: pd.Series, asscalar: bool = False - ) -> Tuple[np.ndarray, pd.Series]: + def eval(self, x: pd.Series, asscalar: bool = False) -> Tuple[np.ndarray, pd.Series]: # _x = x.copy() # preprocessing needs to change the reference x Series so we don't copy here if self.surrogate_model_name == "pfn": - _x_tok, _x, inc_list = self.preprocess_pfn(x.copy()) # IMPORTANT change from vanilla-EI + _x_tok, _x, inc_list = self.preprocess_pfn( + x.copy() + ) # IMPORTANT change from vanilla-EI ei = self.eval_pfn_ei(_x_tok, inc_list) elif self.surrogate_model_name == "deep_gp": - _x, inc_list = self.preprocess_deep_gp(x.copy()) # IMPORTANT change from vanilla-EI + _x, inc_list = self.preprocess_deep_gp( + x.copy() + ) # IMPORTANT change from vanilla-EI ei = self.eval_gp_ei(_x, inc_list) _x = pd.Series(_x, index=np.arange(len(_x))) else: - _x, inc_list = self.preprocess_gp(x.copy()) # IMPORTANT change from vanilla-EI + _x, inc_list = self.preprocess_gp( + x.copy() + ) # IMPORTANT change from vanilla-EI ei = self.eval_gp_ei(_x, inc_list) _x = pd.Series(_x, index=np.arange(len(_x))) - + if ei.is_cuda: ei = ei.cpu() if len(x) > 1 and asscalar: diff --git a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py b/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py index 310a9b21..dfe7a60b 100644 --- a/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py +++ b/src/neps/optimizers/bayesian_optimization/acquisition_samplers/freeze_thaw_sampler.py @@ -8,7 +8,7 @@ import pandas as pd from ....search_spaces.search_space import SearchSpace -from ...multi_fidelity.utils import MFObservedData, continuous_to_tabular +from ...multi_fidelity.utils import MFObservedData from .base_acq_sampler import AcquisitionSampler @@ -23,29 +23,27 @@ def __init__(self, **kwargs): self.pipeline_space = None self.is_tabular = False - def _sample_new( self, index_from: int, n: int = None, ignore_fidelity: bool = False ) -> pd.Series: - n = n if n is not None else self.SAMPLES_TO_DRAW - new_configs = [self.pipeline_space.sample( - patience=self.patience, user_priors=False, ignore_fidelity=ignore_fidelity - ) for _ in range(n)] - - # if self.tabular_space is not None: - # # This function have 3 possible return options: - # # 1. Tabular data is provided then, n configs are sampled from the table - # # 2. Tabular data is not provided and a list of configs is provided then, same list of configs is returned - # # 3. Tabular data is not provided and a single config is provided then, n configs will be sampled randomly - # new_configs=self.tabular_space.sample(index_from=index_from, config=new_configs, n=n) - + new_configs = [ + self.pipeline_space.sample( + patience=self.patience, user_priors=False, ignore_fidelity=ignore_fidelity + ) + for _ in range(n) + ] + return pd.Series( new_configs, index=range(index_from, index_from + len(new_configs)) ) def _sample_new_unique( - self, index_from: int, n: int = None, patience: int = 10, ignore_fidelity: bool=False + self, + index_from: int, + n: int = None, + patience: int = 10, + ignore_fidelity: bool = False, ) -> pd.Series: n = n if n is not None else self.SAMPLES_TO_DRAW assert ( @@ -58,13 +56,17 @@ def _sample_new_unique( # Sample patience times for an unobserved configuration for _ in range(patience): _config = self.pipeline_space.sample( - patience=self.patience, user_priors=False, ignore_fidelity=ignore_fidelity + patience=self.patience, + user_priors=False, + ignore_fidelity=ignore_fidelity, ) # # Convert continuous into tabular if the space is tabular # _config = continuous_to_tabular(_config, self.tabular_space) # Iterate over all observed configs for config in existing_configs: - if _config.is_equal_value(config, include_fidelity=not ignore_fidelity): + if _config.is_equal_value( + config, include_fidelity=not ignore_fidelity + ): # if the sampled config already exists # do the next iteration of patience break @@ -90,36 +92,47 @@ def _sample_new_unique( ) def sample( - self, - acquisition_function=None, - n: int = None, - set_new_sample_fidelity: int | float=None - ) -> list(): + self, + acquisition_function=None, + n: int = None, + set_new_sample_fidelity: int | float = None, + ) -> list(): """Samples a new set and returns the total set of observed + new configs.""" partial_configs = self.observations.get_partial_configs_at_max_seen() new_configs = self._sample_new( index_from=self.observations.next_config_id(), n=n, ignore_fidelity=False ) + def __sample_single_new_tabular(index: int): + """ + A function to use in a list comprehension to slightly speed up + the sampling process when self.SAMPLE_TO_DRAW is large + """ + config = self.pipeline_space.sample( + patience=self.patience, user_priors=False, ignore_fidelity=False + ) + config["id"].value = _new_configs[index] + config.fidelity.value = set_new_sample_fidelity + return config + if self.is_tabular: _n = n if n is not None else self.SAMPLES_TO_DRAW - _partial_ids = set([conf["id"].value for conf in partial_configs]) + _partial_ids = {conf["id"].value for conf in partial_configs} _all_ids = set(self.pipeline_space.custom_grid_table.index.values) # accounting for unseen configs only _n = min(_n, len(_all_ids - _partial_ids)) - _new_configs = np.random.choice(list(_all_ids - _partial_ids), size=_n, replace=False) - new_configs = [self.pipeline_space.sample( - patience=self.patience, user_priors=False, ignore_fidelity=False - ) for _ in range(_n)] - for i, config in enumerate(new_configs): - config["id"].value = _new_configs[i] - config.fidelity.value = self.pipeline_space.fidelity.lower + _new_configs = np.random.choice( + list(_all_ids - _partial_ids), size=_n, replace=False + ) + new_configs = [__sample_single_new_tabular(i) for i in range(_n)] new_configs = pd.Series( new_configs, - index=np.arange(len(partial_configs), len(partial_configs) + len(new_configs)) + index=np.arange( + len(partial_configs), len(partial_configs) + len(new_configs) + ), ) - if set_new_sample_fidelity is not None: + elif set_new_sample_fidelity is not None: for config in new_configs: config.fidelity.value = set_new_sample_fidelity @@ -135,12 +148,8 @@ def sample( # incrementing fidelities multiple times due to pass-by-reference partial_configs = pd.Series(partial_configs_list, index=index_list) - # Set fidelity for new configs - for _, config in new_configs.items(): - config.fidelity.value = config.fidelity.lower - configs = pd.concat([partial_configs, new_configs]) - + return configs def set_state( @@ -155,6 +164,8 @@ def set_state( self.observations = observations self.b_step = b_step self.n = n if n is not None else self.SAMPLES_TO_DRAW - if hasattr(self.pipeline_space, "custom_grid_table") and self.pipeline_space.custom_grid_table is not None: + if ( + hasattr(self.pipeline_space, "custom_grid_table") + and self.pipeline_space.custom_grid_table is not None + ): self.is_tabular = True - diff --git a/src/neps/optimizers/multi_fidelity/dyhpo.py b/src/neps/optimizers/multi_fidelity/dyhpo.py index 52e6a570..f97be3f7 100755 --- a/src/neps/optimizers/multi_fidelity/dyhpo.py +++ b/src/neps/optimizers/multi_fidelity/dyhpo.py @@ -2,7 +2,6 @@ # type: ignore from __future__ import annotations -from copy import deepcopy from typing import Any import numpy as np @@ -19,9 +18,9 @@ ) from ..bayesian_optimization.kernels.get_kernels import get_kernels from .mf_bo import FreezeThawModel, PFNSurrogate + # MFEIDeepModel, MFEIModel -from .utils import MFObservedData, TabularSearchSpace -import pandas as pd +from .utils import MFObservedData class MFEIBO(BaseOptimizer): @@ -96,9 +95,11 @@ def __init__( self._initial_design_size, self._initial_design_budget = self._set_initial_design( initial_design_size, initial_design_budget, self._initial_design_fraction ) + # TODO: Write use cases for these parameters self._model_update_failed = False self.sample_default_first = sample_default_first self.sample_default_at_target = sample_default_at_target + self.surrogate_model_name = surrogate_model self.use_priors = use_priors @@ -121,8 +122,8 @@ def __init__( {} if surrogate_model_args is None else surrogate_model_args ) self._prep_model_args(self.hp_kernels, self.graph_kernels, pipeline_space) - - # TODO: do we create a model policy map? + + # TODO: Better solution than branching based on the surrogate name is needed if surrogate_model in ["deep_gp", "gp"]: model_policy = FreezeThawModel elif surrogate_model == "pfn": @@ -136,11 +137,13 @@ def __init__( surrogate_model=surrogate_model, surrogate_model_args=self.surrogate_model_args, ) - self.acquisition_args = ({} if acquisition_args is None else acquisition_args) - self.acquisition_args.update({ - "pipeline_space": self.pipeline_space, - "surrogate_model_name": self.surrogate_model_name, - }) + self.acquisition_args = {} if acquisition_args is None else acquisition_args + self.acquisition_args.update( + { + "pipeline_space": self.pipeline_space, + "surrogate_model_name": self.surrogate_model_name, + } + ) self.acquisition = instance_from_map( AcquisitionMapping, acquisition, @@ -150,10 +153,9 @@ def __init__( self.acquisition_sampler_args = ( {} if acquisition_sampler_args is None else acquisition_sampler_args ) - self.acquisition_sampler_args.update({ - "patience": self.patience, - "pipeline_space": self.pipeline_space - }) + self.acquisition_sampler_args.update( + {"patience": self.patience, "pipeline_space": self.pipeline_space} + ) self.acquisition_sampler = instance_from_map( AcquisitionSamplerMapping, acquisition_sampler, @@ -161,7 +163,6 @@ def __init__( kwargs=self.acquisition_sampler_args, ) self.count = 0 - self.load_df = None def _prep_model_args(self, hp_kernels, graph_kernels, pipeline_space): if self.surrogate_model_name in ["gp", "gp_hierarchy"]: @@ -177,8 +178,9 @@ def _prep_model_args(self, hp_kernels, graph_kernels, pipeline_space): raise ValueError("No kernels are provided!") # if "vectorial_features" not in self.surrogate_model_args: self.surrogate_model_args["vectorial_features"] = ( - pipeline_space.raw_tabular_space.get_vectorial_dim() - if pipeline_space.has_tabular else pipeline_space.get_vectorial_dim() + pipeline_space.raw_tabular_space.get_vectorial_dim() + if pipeline_space.has_tabular + else pipeline_space.get_vectorial_dim() ) def _set_initial_design( @@ -221,7 +223,9 @@ def _set_initial_design( return _initial_design_size, _initial_design_budget def get_budget_level(self, config: SearchSpace) -> int: - return int(np.ceil((config.fidelity.value - config.fidelity.lower) / self.step_size)) + return int( + np.ceil((config.fidelity.value - config.fidelity.lower) / self.step_size) + ) def get_budget_value(self, budget_level: int | float) -> int | float: if isinstance(self.pipeline_space.fidelity, IntegerParameter): @@ -250,18 +254,20 @@ def total_budget_spent(self) -> int | float: """ if len(self.observed_configs.df) == 0: return 0 - + n_configs = len(self.observed_configs.seen_config_ids) total_budget_level = sum(self.observed_configs.seen_budget_levels) total_initial_budget_spent = n_configs * self.pipeline_space.fidelity.lower - total_budget_spent = (total_initial_budget_spent + - total_budget_level * self.step_size) + total_budget_spent = ( + total_initial_budget_spent + total_budget_level * self.step_size + ) return total_budget_spent def is_init_phase(self, budget_based: bool = True) -> bool: if budget_based: - # TODO: what does this variable and condition do? (karibbov) + # Check if we are still in the initial design phase based on + # either the budget spent so far or the number of configurations evaluated if self.total_budget_spent() < self._initial_design_budget: return True else: @@ -302,7 +308,7 @@ def load_results( ) # TODO: can we do better than keeping a copy of the observed configs? - # TODO: can we not hide this in load_results and have something that pops out + # TODO: can we not hide this in load_results and have something that pops out # more, like a set_state or policy_args self.model_policy.observed_configs = self.observed_configs # fit any model/surrogates @@ -318,18 +324,21 @@ def _get_config_id_split(cls, config_id: str) -> tuple[str, str]: return _config, _budget def _load_previous_observations(self, previous_results): - def index_data_split(config_id: str, config_val): _config_id, _budget_id = MFEIBO._get_config_id_split(config_id) index = int(_config_id), int(_budget_id) - _data = [config_val.config, - self.get_loss(config_val.result), - self.get_learning_curve(config_val.result)] + _data = [ + config_val.config, + self.get_loss(config_val.result), + self.get_learning_curve(config_val.result), + ] return index, _data if len(previous_results) > 0: - index_row = [tuple(index_data_split(config_id, config_val)) - for config_id, config_val in previous_results.items()] + index_row = [ + tuple(index_data_split(config_id, config_val)) + for config_id, config_val in previous_results.items() + ] indices, rows = zip(*index_row) self.observed_configs.add_data(data=list(rows), index=list(indices)) @@ -360,10 +369,10 @@ def _fit_models(self): self.model_policy.set_state(self.pipeline_space, self.surrogate_model_args) self.model_policy.update_model() self.acquisition.set_state( - self.pipeline_space, + self.pipeline_space, self.model_policy.surrogate_model, self.observed_configs, - self.step_size + self.step_size, ) self.acquisition_sampler.set_state( self.pipeline_space, self.observed_configs, self.step_size @@ -439,8 +448,10 @@ def get_config_and_ids( # pylint: disable=no-self-use # NOTE: len(samples) need not be equal to len(_samples) as `samples` contain # all (partials + new) configurations obtained from the sampler, but # in `_samples`, configs are removed that have reached maximum epochs allowed - # NOTE: `samples` and `_samples` should share the same index values, hence, + # NOTE: `samples` and `_samples` should share the same index values, hence, # avoid using `.iloc` and work with `.loc` on pandas DataFrame/Series + + # Is this "config = _samples.loc[_config_id]"? config = samples.loc[_config_id] config.fidelity.value = _samples.loc[_config_id].fidelity.value # generating correct IDs @@ -448,9 +459,6 @@ def get_config_and_ids( # pylint: disable=no-self-use config_id = f"{_config_id}_{self.get_budget_level(config)}" previous_config_id = f"{_config_id}_{self.get_budget_level(config) - 1}" else: - config_id = ( - f"{self.observed_configs.next_config_id()}_{self.get_budget_level(config)}" - ) - curr_id = len(self.observed_configs.df) + config_id = f"{self.observed_configs.next_config_id()}_{self.get_budget_level(config)}" return config.hp_values(), config_id, previous_config_id diff --git a/src/neps/optimizers/multi_fidelity/mf_bo.py b/src/neps/optimizers/multi_fidelity/mf_bo.py index 4c845e93..3cf191bd 100755 --- a/src/neps/optimizers/multi_fidelity/mf_bo.py +++ b/src/neps/optimizers/multi_fidelity/mf_bo.py @@ -2,6 +2,7 @@ from __future__ import annotations from copy import deepcopy + import numpy as np import pandas as pd import torch @@ -15,8 +16,8 @@ class MFBOBase: - """ Designed to work with model-based search on SH-based multi-fidelity algorithms. - + """Designed to work with model-based search on SH-based multi-fidelity algorithms. + Requires certain strict assumptions about fidelities and rung maps. """ @@ -182,7 +183,7 @@ def sample_new_config( class FreezeThawModel: - """ Designed to work with model search in unit step multi-fidelity algorithms.""" + """Designed to work with model search in unit step multi-fidelity algorithms.""" def __init__( self, @@ -240,19 +241,24 @@ def _fit(self, train_x, train_y, train_lcs): raise ValueError( f"Surrogate model {self.surrogate_model_name} not supported!" ) - + def _predict(self, test_x, test_lcs): if self.surrogate_model_name in ["gp", "gp_hierarchy"]: - self.surrogate_model.predict(test_x) + return self.surrogate_model.predict(test_x) elif self.surrogate_model_name in ["deep_gp", "pfn"]: - self.surrogate_model.predict(test_x, test_lcs) + return self.surrogate_model.predict(test_x, test_lcs) else: # check neps/optimizers/bayesian_optimization/models/__init__.py for options raise ValueError( f"Surrogate model {self.surrogate_model_name} not supported!" ) - def set_state(self, pipeline_space, surrogate_model_args, **kwargs): + def set_state( + self, + pipeline_space, + surrogate_model_args, + **kwargs, # pylint: disable=unused-argument + ): self.pipeline_space = pipeline_space self.surrogate_model_args = ( surrogate_model_args if surrogate_model_args is not None else {} @@ -260,9 +266,9 @@ def set_state(self, pipeline_space, surrogate_model_args, **kwargs): # only to handle tabular spaces if self.pipeline_space.has_tabular: if self.surrogate_model_name in ["deep_gp", "pfn"]: - self.surrogate_model_args.update({ - "pipeline_space": self.pipeline_space.raw_tabular_space - }) + self.surrogate_model_args.update( + {"pipeline_space": self.pipeline_space.raw_tabular_space} + ) # instantiate the surrogate model, again, with the new pipeline space self.surrogate_model = instance_from_map( SurrogateModelMapping, @@ -289,12 +295,13 @@ def update_model(self, train_x=None, train_y=None, pending_x=None, decay_t=None) class PFNSurrogate(FreezeThawModel): """Special class to deal with PFN surrogate model and freeze-thaw acquisition.""" + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.train_x = None self.train_y = None - def _fit(self, *args): + def _fit(self, *args): # pylint: disable=unused-argument assert self.surrogate_model_name == "pfn" self.preprocess_training_set() self.surrogate_model.fit(self.train_x, self.train_y) @@ -310,7 +317,7 @@ def preprocess_training_set(self): _configs = map_real_hyperparameters_from_tabular_ids( pd.Series(_configs, index=_idxs), self.pipeline_space ).values - + device = self.surrogate_model.device # TODO: fix or make consistent with `tokenize`` configs, idxs, performances = self.observed_configs.get_tokenized_data( @@ -326,9 +333,9 @@ def preprocess_test_set(self, test_x): new_idxs = np.arange(_len, len(test_x)) base_fidelity = np.array([1] * len(new_idxs)) - new_token_ids = np.hstack(( - new_idxs.T.reshape(-1, 1), base_fidelity.T.reshape(-1, 1) - )) + new_token_ids = np.hstack( + (new_idxs.T.reshape(-1, 1), base_fidelity.T.reshape(-1, 1)) + ) # the following operation takes each element in the array and stacks it vertically # in this case, should convert a (n,) array to (n, 2) by flattening the elements existing_token_ids = np.vstack(self.observed_configs.token_ids).astype(int) diff --git a/src/neps/optimizers/multi_fidelity/utils.py b/src/neps/optimizers/multi_fidelity/utils.py index 4c1be7d3..f1d359a6 100644 --- a/src/neps/optimizers/multi_fidelity/utils.py +++ b/src/neps/optimizers/multi_fidelity/utils.py @@ -1,3 +1,4 @@ +# type: ignore from __future__ import annotations from typing import Any, Sequence @@ -28,8 +29,9 @@ def continuous_to_tabular( return result + def normalize_vectorize_config( - config: SearchSpace, ignore_fidelity: bool=True + config: SearchSpace, ignore_fidelity: bool = True ) -> np.ndarray: _new_vector = [] for _, hp_list in config.get_normalized_hp_categories(ignore_fidelity).items(): @@ -93,7 +95,7 @@ def error_condition(self): @property def seen_config_ids(self) -> list: return self.df.index.levels[0].to_list() - + @property def seen_budget_levels(self) -> list: # Considers pending and error budgets as seen @@ -239,6 +241,9 @@ def get_partial_configs_at_max_seen(self): return self.reduce_to_max_seen_budgets()[self.config_col] def extract_learning_curve(self, config_id: int, budget_id: int) -> list[float]: + # reduce budget_id to discount the current validation loss + # both during training and prediction phase + budget_id = max(0, budget_id - 1) if self.lc_col_name in self.df.columns: lc = self.df.loc[(config_id, budget_id), self.lc_col_name] else: @@ -273,7 +278,7 @@ def get_tokenized_data(self, df: pd.DataFrame): configs = df.config.values configs = np.array([normalize_vectorize_config(c) for c in configs]) - return configs, idxs, performances + return configs, idxs, performances def tokenize(self, df: pd.DataFrame, as_tensor: bool = False): """Function to format data for PFN.""" @@ -287,75 +292,12 @@ def tokenize(self, df: pd.DataFrame, as_tensor: bool = False): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") data = torch.Tensor(data).to(device) return data - + @property def token_ids(self) -> np.ndarray: return self.df.index.values - -class TabularSearchSpace: - tabular = True - - def __init__(self, tabular_df: pd.DataFrame, base_search_space: SearchSpace): - # IMPORTANT: TO WORK WITH TABULAR SEARCH SPACE OVER MULTIPLE RESTARTS (OWERWRITE=FALSE) - # SEEDS MUST BE SET MANUALLY OUTSIDE NEPS OPTIMIZERS - # Otherwise generated permutations will not be consistent over different runs - if tabular_df is not None: - place_holder_config = base_search_space.sample() - self.table = TabularSearchSpace.convert_tabular(tabular_df, place_holder_config) - self.index_permutation = np.random.permutation(self.table.index) - else: - self.tabular = False - - @staticmethod - def __build_min_fidelity_config(row, config: SearchSpace): - result = config.copy() - for hp_name in config.keys(): - if not config[hp_name].is_fidelity: - # dynamic type casting to the target value - # this is only necessary if the dtypes of the dataframe is not set correctly - # otherwise: value = row[hp_name] # is sufficient - value = type(result[hp_name].value)(row[hp_name]) - result[hp_name].value = value - else: - result[hp_name].value = result.fidelity.lower - return result - - @staticmethod - def convert_tabular(tabular_benchmark: pd.DataFrame, - placeholder_config: SearchSpace): - - config_keys = [] - for hp_name, hp in placeholder_config.items(): - if not hp.is_fidelity: - config_keys.append(hp_name) - - df = tabular_benchmark.loc[:, config_keys].copy(deep=True) - samples = df.groupby(level=0).first() - - samples["configs"] = samples.apply( - lambda x: TabularSearchSpace.__build_min_fidelity_config(x, config=placeholder_config), axis=1) - - samples.index = samples.index.astype(int, copy=True) - samples.sort_index(inplace=True) - - samples.drop(config_keys, inplace=True, axis=1) - - return samples - - def sample(self, index_from: int, - config: SearchSpace | List[SearchSpace] | None = None, - n: int | None = None): - n = n if n is not None else 1 - if self.tabular: - return self.table.loc[self.index_permutation[index_from:index_from + n], :].values.tolist() - elif n > 1 and not isinstance(config, list): - return [config.sample() for _ in range(n)] - else: - return config - - if __name__ == "__main__": # TODO: Either delete these or convert them to tests (karibbov) """