diff --git a/ax/core/map_data.py b/ax/core/map_data.py index b55e57344c8..dfbdcea50c1 100644 --- a/ax/core/map_data.py +++ b/ax/core/map_data.py @@ -184,7 +184,10 @@ def from_multiple_map_data( unique_map_key_infos = [] for mki in (mki for datum in data for mki in datum.map_key_infos): if any( - mki.key == unique.key and mki.default_value != unique.default_value + mki.key == unique.key + and not np.isclose( + mki.default_value, unique.default_value, equal_nan=True + ) for unique in unique_map_key_infos ): logger.warning(f"MapKeyInfo conflict for {mki.key}, eliding {mki}.") diff --git a/ax/preview/api/tests/test_client.py b/ax/preview/api/tests/test_client.py index 1f83295ac1d..260d269ee29 100644 --- a/ax/preview/api/tests/test_client.py +++ b/ax/preview/api/tests/test_client.py @@ -16,7 +16,7 @@ from ax.core.experiment import Experiment from ax.core.formatting_utils import DataType from ax.core.map_data import MapData -from ax.core.metric import Metric +from ax.core.map_metric import MapMetric from ax.core.objective import MultiObjective, Objective, ScalarizedObjective from ax.core.optimization_config import OptimizationConfig from ax.core.outcome_constraint import ComparisonOp, OutcomeConstraint @@ -162,10 +162,10 @@ def test_configure_optimization(self) -> None: self.assertEqual( client._experiment.optimization_config, OptimizationConfig( - objective=Objective(metric=Metric(name="ne"), minimize=True), + objective=Objective(metric=MapMetric(name="ne"), minimize=True), outcome_constraints=[ OutcomeConstraint( - metric=Metric(name="qps"), + metric=MapMetric(name="qps"), op=ComparisonOp.GEQ, bound=0.0, relative=False, @@ -261,7 +261,7 @@ def test_configure_metric(self) -> None: client.configure_optimization( objective="foo", ) - client._experiment.add_tracking_metric(metric=Metric("custom")) + client._experiment.add_tracking_metric(metric=MapMetric("custom")) client.configure_metrics(metrics=[custom_metric]) self.assertEqual( diff --git a/ax/preview/api/utils/instantiation/from_string.py b/ax/preview/api/utils/instantiation/from_string.py index ac0dd827c3b..f3230799df1 100644 --- a/ax/preview/api/utils/instantiation/from_string.py +++ b/ax/preview/api/utils/instantiation/from_string.py @@ -7,7 +7,8 @@ from typing import Sequence -from ax.core.metric import Metric +from ax.core.map_metric import MapMetric + from ax.core.objective import MultiObjective, Objective, ScalarizedObjective from ax.core.optimization_config import ( MultiObjectiveOptimizationConfig, @@ -181,7 +182,7 @@ def parse_outcome_constraint(constraint_str: str) -> OutcomeConstraint: term, coefficient = next(iter(constraint_dict.items())) return OutcomeConstraint( - metric=Metric(name=term), + metric=MapMetric(name=term), op=ComparisonOp.LEQ if coefficient > 0 else ComparisonOp.GEQ, bound=bound / coefficient, relative=is_relative, @@ -189,7 +190,7 @@ def parse_outcome_constraint(constraint_str: str) -> OutcomeConstraint: names, coefficients = zip(*constraint_dict.items()) return ScalarizedOutcomeConstraint( - metrics=[Metric(name=name) for name in names], + metrics=[MapMetric(name=name) for name in names], op=ComparisonOp.LEQ, weights=[*coefficients], bound=bound, @@ -206,7 +207,7 @@ def _create_single_objective(expression: Expr) -> Objective: # If the expression is a just a Symbol it represents a single metric objective if isinstance(expression, Symbol): - return Objective(metric=Metric(name=str(expression.name)), minimize=False) + return Objective(metric=MapMetric(name=str(expression.name)), minimize=False) # If the expression is a Mul it likely represents a single metric objective but # some additional validation is required @@ -221,13 +222,13 @@ def _create_single_objective(expression: Expr) -> Objective: # the sign from the coefficient rather than its value minimize = bool(expression.as_coefficient(symbol) < 0) - return Objective(metric=Metric(name=str(symbol)), minimize=minimize) + return Objective(metric=MapMetric(name=str(symbol)), minimize=minimize) # If the expression is an Add it represents a scalarized objective elif isinstance(expression, Add): names, coefficients = zip(*expression.as_coefficients_dict().items()) return ScalarizedObjective( - metrics=[Metric(name=str(name)) for name in names], + metrics=[MapMetric(name=str(name)) for name in names], weights=[float(coefficient) for coefficient in coefficients], minimize=False, ) diff --git a/ax/preview/api/utils/instantiation/tests/test_from_string.py b/ax/preview/api/utils/instantiation/tests/test_from_string.py index fc897a1dc0d..995a658ef8f 100644 --- a/ax/preview/api/utils/instantiation/tests/test_from_string.py +++ b/ax/preview/api/utils/instantiation/tests/test_from_string.py @@ -5,7 +5,7 @@ # pyre-strict -from ax.core.metric import Metric +from ax.core.map_metric import MapMetric from ax.core.objective import MultiObjective, Objective, ScalarizedObjective from ax.core.optimization_config import ( MultiObjectiveOptimizationConfig, @@ -34,7 +34,7 @@ def test_optimization_config_from_string(self) -> None: self.assertEqual( only_objective, OptimizationConfig( - objective=Objective(metric=Metric(name="ne"), minimize=False), + objective=Objective(metric=MapMetric(name="ne"), minimize=False), ), ) @@ -44,10 +44,10 @@ def test_optimization_config_from_string(self) -> None: self.assertEqual( with_constraints, OptimizationConfig( - objective=Objective(metric=Metric(name="ne"), minimize=False), + objective=Objective(metric=MapMetric(name="ne"), minimize=False), outcome_constraints=[ OutcomeConstraint( - metric=Metric(name="qps"), + metric=MapMetric(name="qps"), op=ComparisonOp.GEQ, bound=0.0, relative=False, @@ -65,13 +65,13 @@ def test_optimization_config_from_string(self) -> None: MultiObjectiveOptimizationConfig( objective=MultiObjective( objectives=[ - Objective(metric=Metric(name="ne"), minimize=True), - Objective(metric=Metric(name="qps"), minimize=False), + Objective(metric=MapMetric(name="ne"), minimize=True), + Objective(metric=MapMetric(name="qps"), minimize=False), ] ), outcome_constraints=[ OutcomeConstraint( - metric=Metric(name="flops"), + metric=MapMetric(name="flops"), op=ComparisonOp.LEQ, bound=1000000.0, relative=False, @@ -79,7 +79,7 @@ def test_optimization_config_from_string(self) -> None: ], objective_thresholds=[ ObjectiveThreshold( - metric=Metric(name="qps"), + metric=MapMetric(name="qps"), op=ComparisonOp.GEQ, bound=1000.0, relative=False, @@ -123,13 +123,13 @@ def test_parse_paramter_constraint(self) -> None: def test_parse_objective(self) -> None: single_objective = parse_objective(objective_str="ne") self.assertEqual( - single_objective, Objective(metric=Metric(name="ne"), minimize=False) + single_objective, Objective(metric=MapMetric(name="ne"), minimize=False) ) maximize_single_objective = parse_objective(objective_str="-qps") self.assertEqual( maximize_single_objective, - Objective(metric=Metric(name="qps"), minimize=True), + Objective(metric=MapMetric(name="qps"), minimize=True), ) scalarized_objective = parse_objective( @@ -138,7 +138,11 @@ def test_parse_objective(self) -> None: self.assertEqual( scalarized_objective, ScalarizedObjective( - metrics=[Metric(name="ne1"), Metric(name="ne2"), Metric(name="ne3")], + metrics=[ + MapMetric(name="ne1"), + MapMetric(name="ne2"), + MapMetric(name="ne3"), + ], weights=[0.5, 0.3, 0.2], minimize=False, ), @@ -149,8 +153,8 @@ def test_parse_objective(self) -> None: multiobjective, MultiObjective( objectives=[ - Objective(metric=Metric(name="ne"), minimize=False), - Objective(metric=Metric(name="qps"), minimize=True), + Objective(metric=MapMetric(name="ne"), minimize=False), + Objective(metric=MapMetric(name="qps"), minimize=True), ] ), ) @@ -163,7 +167,7 @@ def test_parse_outcome_constraint(self) -> None: self.assertEqual( constraint, OutcomeConstraint( - metric=Metric(name="flops"), + metric=MapMetric(name="flops"), op=ComparisonOp.LEQ, bound=1000000.0, relative=False, @@ -174,7 +178,7 @@ def test_parse_outcome_constraint(self) -> None: self.assertEqual( flipped_sign, OutcomeConstraint( - metric=Metric(name="flops"), + metric=MapMetric(name="flops"), op=ComparisonOp.GEQ, bound=1000000.0, relative=False, @@ -185,7 +189,7 @@ def test_parse_outcome_constraint(self) -> None: self.assertEqual( relative, OutcomeConstraint( - metric=Metric(name="flops"), + metric=MapMetric(name="flops"), op=ComparisonOp.LEQ, bound=105.0, relative=True, @@ -198,7 +202,7 @@ def test_parse_outcome_constraint(self) -> None: self.assertEqual( scalarized, ScalarizedOutcomeConstraint( - metrics=[Metric(name="flops1"), Metric(name="flops2")], + metrics=[MapMetric(name="flops1"), MapMetric(name="flops2")], weights=[0.5, 0.3], op=ComparisonOp.LEQ, bound=1000000.0,