Construct MapMetrics by default (#3192)

Summary:

As titled. This should negate some logspam.

Also change MapKeyInfo fusion to consider nan == nan because previously MapKeyInfo(step, nan) fusing with itself would look like a conflict

Reviewed By: saitcakmak

Differential Revision: D67412730

ax/core/map_data.py

@@ -184,7 +184,10 @@ def from_multiple_map_data(
         unique_map_key_infos = []
         for mki in (mki for datum in data for mki in datum.map_key_infos):
             if any(
-                mki.key == unique.key and mki.default_value != unique.default_value
+                mki.key == unique.key
+                and not np.isclose(
+                    mki.default_value, unique.default_value, equal_nan=True
+                )
                 for unique in unique_map_key_infos
             ):
                 logger.warning(f"MapKeyInfo conflict for {mki.key}, eliding {mki}.")

ax/preview/api/tests/test_client.py

@@ -16,7 +16,7 @@
 from ax.core.experiment import Experiment
 from ax.core.formatting_utils import DataType
 from ax.core.map_data import MapData
-from ax.core.metric import Metric
+from ax.core.map_metric import MapMetric
 from ax.core.objective import MultiObjective, Objective, ScalarizedObjective
 from ax.core.optimization_config import OptimizationConfig
 from ax.core.outcome_constraint import ComparisonOp, OutcomeConstraint
@@ -162,10 +162,10 @@ def test_configure_optimization(self) -> None:
         self.assertEqual(
             client._experiment.optimization_config,
             OptimizationConfig(
-                objective=Objective(metric=Metric(name="ne"), minimize=True),
+                objective=Objective(metric=MapMetric(name="ne"), minimize=True),
                 outcome_constraints=[
                     OutcomeConstraint(
-                        metric=Metric(name="qps"),
+                        metric=MapMetric(name="qps"),
                         op=ComparisonOp.GEQ,
                         bound=0.0,
                         relative=False,
@@ -261,7 +261,7 @@ def test_configure_metric(self) -> None:
         client.configure_optimization(
             objective="foo",
         )
-        client._experiment.add_tracking_metric(metric=Metric("custom"))
+        client._experiment.add_tracking_metric(metric=MapMetric("custom"))
         client.configure_metrics(metrics=[custom_metric])
 
         self.assertEqual(

ax/preview/api/utils/instantiation/from_string.py

@@ -8,7 +8,8 @@
 import re
 from typing import Sequence
 
-from ax.core.metric import Metric
+from ax.core.map_metric import MapMetric
+
 from ax.core.objective import MultiObjective, Objective, ScalarizedObjective
 from ax.core.optimization_config import (
     MultiObjectiveOptimizationConfig,
@@ -184,15 +185,15 @@ def parse_outcome_constraint(constraint_str: str) -> OutcomeConstraint:
         term, coefficient = next(iter(constraint_dict.items()))
 
         return OutcomeConstraint(
-            metric=Metric(name=_unsanitize_dot(term)),
+            metric=MapMetric(name=_unsanitize_dot(term)),
             op=ComparisonOp.LEQ if coefficient > 0 else ComparisonOp.GEQ,
             bound=bound / coefficient,
             relative=is_relative,
         )
 
     names, coefficients = zip(*constraint_dict.items())
     return ScalarizedOutcomeConstraint(
-        metrics=[Metric(name=_unsanitize_dot(name)) for name in names],
+        metrics=[MapMetric(name=_unsanitize_dot(name)) for name in names],
         op=ComparisonOp.LEQ,
         weights=[*coefficients],
         bound=bound,
@@ -210,7 +211,7 @@ def _create_single_objective(expression: Expr) -> Objective:
     # If the expression is a just a Symbol it represents a single metric objective
     if isinstance(expression, Symbol):
         return Objective(
-            metric=Metric(name=_unsanitize_dot(str(expression.name))), minimize=False
+            metric=MapMetric(name=_unsanitize_dot(str(expression.name))), minimize=False
         )
 
     # If the expression is a Mul it likely represents a single metric objective but
@@ -227,14 +228,14 @@ def _create_single_objective(expression: Expr) -> Objective:
         minimize = bool(expression.as_coefficient(symbol) < 0)
 
         return Objective(
-            metric=Metric(name=_unsanitize_dot(str(symbol))), minimize=minimize
+            metric=MapMetric(name=_unsanitize_dot(str(symbol))), minimize=minimize
         )
 
     # If the expression is an Add it represents a scalarized objective
     elif isinstance(expression, Add):
         names, coefficients = zip(*expression.as_coefficients_dict().items())
         return ScalarizedObjective(
-            metrics=[Metric(name=_unsanitize_dot(str(name))) for name in names],
+            metrics=[MapMetric(name=_unsanitize_dot(str(name))) for name in names],
             weights=[float(coefficient) for coefficient in coefficients],
             minimize=False,
         )

ax/preview/api/utils/instantiation/tests/test_from_string.py

@@ -5,7 +5,7 @@
 
 # pyre-strict
 
-from ax.core.metric import Metric
+from ax.core.map_metric import MapMetric
 from ax.core.objective import MultiObjective, Objective, ScalarizedObjective
 from ax.core.optimization_config import (
     MultiObjectiveOptimizationConfig,
@@ -35,7 +35,7 @@ def test_optimization_config_from_string(self) -> None:
         self.assertEqual(
             only_objective,
             OptimizationConfig(
-                objective=Objective(metric=Metric(name="ne"), minimize=False),
+                objective=Objective(metric=MapMetric(name="ne"), minimize=False),
             ),
         )
 
@@ -45,10 +45,10 @@ def test_optimization_config_from_string(self) -> None:
         self.assertEqual(
             with_constraints,
             OptimizationConfig(
-                objective=Objective(metric=Metric(name="ne"), minimize=False),
+                objective=Objective(metric=MapMetric(name="ne"), minimize=False),
                 outcome_constraints=[
                     OutcomeConstraint(
-                        metric=Metric(name="qps"),
+                        metric=MapMetric(name="qps"),
                         op=ComparisonOp.GEQ,
                         bound=0.0,
                         relative=False,
@@ -66,21 +66,21 @@ def test_optimization_config_from_string(self) -> None:
             MultiObjectiveOptimizationConfig(
                 objective=MultiObjective(
                     objectives=[
-                        Objective(metric=Metric(name="ne"), minimize=True),
-                        Objective(metric=Metric(name="qps"), minimize=False),
+                        Objective(metric=MapMetric(name="ne"), minimize=True),
+                        Objective(metric=MapMetric(name="qps"), minimize=False),
                     ]
                 ),
                 outcome_constraints=[
                     OutcomeConstraint(
-                        metric=Metric(name="flops"),
+                        metric=MapMetric(name="flops"),
                         op=ComparisonOp.LEQ,
                         bound=1000000.0,
                         relative=False,
                     )
                 ],
                 objective_thresholds=[
                     ObjectiveThreshold(
-                        metric=Metric(name="qps"),
+                        metric=MapMetric(name="qps"),
                         op=ComparisonOp.GEQ,
                         bound=1000.0,
                         relative=False,
@@ -124,13 +124,13 @@ def test_parse_paramter_constraint(self) -> None:
     def test_parse_objective(self) -> None:
         single_objective = parse_objective(objective_str="ne")
         self.assertEqual(
-            single_objective, Objective(metric=Metric(name="ne"), minimize=False)
+            single_objective, Objective(metric=MapMetric(name="ne"), minimize=False)
         )
 
         maximize_single_objective = parse_objective(objective_str="-qps")
         self.assertEqual(
             maximize_single_objective,
-            Objective(metric=Metric(name="qps"), minimize=True),
+            Objective(metric=MapMetric(name="qps"), minimize=True),
         )
 
         scalarized_objective = parse_objective(
@@ -139,7 +139,11 @@ def test_parse_objective(self) -> None:
         self.assertEqual(
             scalarized_objective,
             ScalarizedObjective(
-                metrics=[Metric(name="ne1"), Metric(name="ne2"), Metric(name="ne3")],
+                metrics=[
+                    MapMetric(name="ne1"),
+                    MapMetric(name="ne2"),
+                    MapMetric(name="ne3"),
+                ],
                 weights=[0.5, 0.3, 0.2],
                 minimize=False,
             ),
@@ -150,8 +154,8 @@ def test_parse_objective(self) -> None:
             multiobjective,
             MultiObjective(
                 objectives=[
-                    Objective(metric=Metric(name="ne"), minimize=False),
-                    Objective(metric=Metric(name="qps"), minimize=True),
+                    Objective(metric=MapMetric(name="ne"), minimize=False),
+                    Objective(metric=MapMetric(name="qps"), minimize=True),
                 ]
             ),
         )
@@ -164,7 +168,7 @@ def test_parse_outcome_constraint(self) -> None:
         self.assertEqual(
             constraint,
             OutcomeConstraint(
-                metric=Metric(name="flops"),
+                metric=MapMetric(name="flops"),
                 op=ComparisonOp.LEQ,
                 bound=1000000.0,
                 relative=False,
@@ -175,7 +179,7 @@ def test_parse_outcome_constraint(self) -> None:
         self.assertEqual(
             flipped_sign,
             OutcomeConstraint(
-                metric=Metric(name="flops"),
+                metric=MapMetric(name="flops"),
                 op=ComparisonOp.GEQ,
                 bound=1000000.0,
                 relative=False,
@@ -186,7 +190,7 @@ def test_parse_outcome_constraint(self) -> None:
         self.assertEqual(
             relative,
             OutcomeConstraint(
-                metric=Metric(name="flops"),
+                metric=MapMetric(name="flops"),
                 op=ComparisonOp.LEQ,
                 bound=105.0,
                 relative=True,
@@ -199,7 +203,7 @@ def test_parse_outcome_constraint(self) -> None:
         self.assertEqual(
             scalarized,
             ScalarizedOutcomeConstraint(
-                metrics=[Metric(name="flops1"), Metric(name="flops2")],
+                metrics=[MapMetric(name="flops1"), MapMetric(name="flops2")],
                 weights=[0.5, 0.3],
                 op=ComparisonOp.LEQ,
                 bound=1000000.0,