Skip to content

Commit

Permalink
Remove best point properties from GSInterface (#1990)
Browse files Browse the repository at this point in the history
Summary:
Pull Request resolved: #1990

Probably to be reimplemented at a later time

Reviewed By: saitcakmak

Differential Revision: D51307864

fbshipit-source-id: 7730d004364d03a06deadbee83b1ac3af6f514db
  • Loading branch information
Daniel Cohen authored and facebook-github-bot committed Nov 15, 2023
1 parent e51d48b commit 64446e3
Showing 1 changed file with 0 additions and 99 deletions.
99 changes: 0 additions & 99 deletions ax/core/generation_strategy_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,102 +114,3 @@ def last_generator_run(self) -> Optional[GeneratorRun]:
"""
# Used to restore current model when decoding a serialized GS.
return self._generator_runs[-1] if self._generator_runs else None

@abstractmethod
def get_pareto_optimal_parameters(
self,
experiment: Experiment,
optimization_config: Optional[OptimizationConfig] = None,
trial_indices: Optional[Iterable[int]] = None,
use_model_predictions: bool = True,
) -> Dict[int, Tuple[TParameterization, TModelPredictArm]]:
"""Identifies the best parameterizations tried in the experiment so far,
using model predictions if ``use_model_predictions`` is true and using
observed values from the experiment otherwise. By default, uses model
predictions to account for observation noise.
NOTE: The format of this method's output is as follows:
{ trial_index --> (parameterization, (means, covariances) }, where means
are a dictionary of form { metric_name --> metric_mean } and covariances
are a nested dictionary of form
{ one_metric_name --> { another_metric_name: covariance } }.
Args:
experiment: Experiment, from which to find Pareto-optimal arms.
optimization_config: Optimization config to use in place of the one stored
on the experiment.
trial_indices: Indices of trials for which to retrieve data. If None will
retrieve data from all available trials.
use_model_predictions: Whether to extract the Pareto frontier using
model predictions or directly observed values. If ``True``,
the metric means and covariances in this method's output will
also be based on model predictions and may differ from the
observed values.
Returns:
A mapping from trial index to the tuple of:
- the parameterization of the arm in that trial,
- two-item tuple of metric means dictionary and covariance matrix
(model-predicted if ``use_model_predictions=True`` and observed
otherwise).
"""
pass

@abstractmethod
def get_best_trial(
self,
experiment: Experiment,
optimization_config: Optional[OptimizationConfig] = None,
trial_indices: Optional[Iterable[int]] = None,
) -> Optional[Tuple[int, TParameterization, Optional[TModelPredictArm]]]:
"""Given an experiment, returns the best predicted parameterization and
corresponding prediction based on the most recent Trial with predictions.
If no trials have predictions returns None.
Only some models return predictions. For instance GPEI does
while Sobol does not.
TModelPredictArm is of the form:
({metric_name: mean}, {metric_name_1: {metric_name_2: cov_1_2}})
Args:
experiment: Experiment, on which to identify best raw objective arm.
optimization_config: Optimization config to use in place of the one stored
on the experiment.
trial_indices: Indices of trials for which to retrieve data. If None will
retrieve data from all available trials.
Returns:
Tuple of trial index, parameterization, and model predictions for it."""
pass

@abstractmethod
def get_hypervolume(
self,
experiment: Experiment,
optimization_config: Optional[MultiObjectiveOptimizationConfig] = None,
trial_indices: Optional[Iterable[int]] = None,
) -> float:
"""Calculate hypervolume of a pareto frontier based on the posterior means of
given observation features.
Given a model and features to evaluate calculate the hypervolume of the pareto
frontier formed from their predicted outcomes.
Args:
modelbridge: Modelbridge used to predict metrics outcomes.
optimization_config: Optimization config
selected_metrics: If specified, hypervolume will only be evaluated on
the specified subset of metrics. Otherwise, all metrics will be used.
trial_indices: Indices of trials for which to retrieve data. If None will
retrieve data from all available trials.
Returns:
calculated hypervolume.
"""
pass

@abstractproperty
def num_baseline_trials(self) -> int:
"""Number of trials generated by the first step in this generation strategy."""
pass

0 comments on commit 64446e3

Please sign in to comment.