From ba00cdca1f3fa2a6447edb9173c692c269acb08b Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Mon, 14 Aug 2023 12:18:59 -0400 Subject: [PATCH 1/8] invalid_target_data_check added with errors --- checkmates/data_checks/__init__.py | 1 + .../checks/invalid_target_data_check.py | 448 ++++++++++ checkmates/exceptions/__init__.py | 2 + checkmates/exceptions/exceptions.py | 7 + checkmates/objectives/__init__.py | 11 + checkmates/objectives/objective_base.py | 217 +++++ checkmates/objectives/regression_objective.py | 10 + checkmates/objectives/standard_metrics.py | 100 +++ checkmates/objectives/utils.py | 139 +++ checkmates/utils/gen_utils.py | 23 + tests/conftest.py | 6 + .../test_invalid_target_data_check.py | 820 ++++++++++++++++++ 12 files changed, 1784 insertions(+) create mode 100644 checkmates/data_checks/checks/invalid_target_data_check.py create mode 100644 checkmates/objectives/__init__.py create mode 100644 checkmates/objectives/objective_base.py create mode 100644 checkmates/objectives/regression_objective.py create mode 100644 checkmates/objectives/standard_metrics.py create mode 100644 checkmates/objectives/utils.py create mode 100644 tests/data_checks_tests/test_invalid_target_data_check.py diff --git a/checkmates/data_checks/__init__.py b/checkmates/data_checks/__init__.py index 1583530..29da738 100644 --- a/checkmates/data_checks/__init__.py +++ b/checkmates/data_checks/__init__.py @@ -35,6 +35,7 @@ from checkmates.data_checks.checks.sparsity_data_check import SparsityDataCheck from checkmates.data_checks.checks.datetime_format_data_check import DateTimeFormatDataCheck from checkmates.data_checks.checks.multicollinearity_data_check import MulticollinearityDataCheck +from checkmates.data_checks.checks.invalid_target_data_check import InvalidTargetDataCheck diff --git a/checkmates/data_checks/checks/invalid_target_data_check.py b/checkmates/data_checks/checks/invalid_target_data_check.py new file mode 100644 index 0000000..fc92625 --- /dev/null +++ b/checkmates/data_checks/checks/invalid_target_data_check.py @@ -0,0 +1,448 @@ +"""Data check that checks if the target data contains missing or invalid values.""" +import woodwork as ww + +from checkmates.data_checks import ( + DataCheck, + DataCheckActionCode, + DataCheckActionOption, + DataCheckError, + DataCheckMessageCode, + DataCheckWarning, + DCAOParameterType, +) +from checkmates.objectives import get_objective +from checkmates.problem_types import ( + handle_problem_types, + is_binary, + is_multiclass, + is_regression, +) +from checkmates.utils.woodwork_utils import infer_feature_types, numeric_and_boolean_ww + + +class InvalidTargetDataCheck(DataCheck): + """Check if the target data is considered invalid. + + Target data is considered invalid if: + - Target is None. + - Target has NaN or None values. + - Target is of an unsupported Woodwork logical type. + - Target and features have different lengths or indices. + - Target does not have enough instances of a class in a classification problem. + - Target does not contain numeric data for regression problems. + + Args: + problem_type (str or ProblemTypes): The specific problem type to data check for. + e.g. 'binary', 'multiclass', 'regression, 'time series regression' + objective (str or ObjectiveBase): Name or instance of the objective class. + n_unique (int): Number of unique target values to store when problem type is binary and target + incorrectly has more than 2 unique values. Non-negative integer. If None, stores all unique values. Defaults to 100. + null_strategy (str): The type of action option that should be returned if the target is partially null. + The options are `impute` and `drop` (default). + `impute` - Will return a `DataCheckActionOption` for imputing the target column. + `drop` - Will return a `DataCheckActionOption` for dropping the null rows in the target column. + """ + + multiclass_continuous_threshold = 0.05 + + def __init__(self, problem_type, objective, n_unique=100, null_strategy="drop"): + self.problem_type = handle_problem_types(problem_type) + self.objective = get_objective(objective) + if n_unique is not None and n_unique <= 0: + raise ValueError("`n_unique` must be a non-negative integer value.") + self.n_unique = n_unique + if null_strategy is None or null_strategy.lower() not in ["impute", "drop"]: + raise ValueError( + "The acceptable values for 'null_strategy' are 'impute' and 'drop'.", + ) + self.null_strategy = null_strategy + + def validate(self, X, y): + """Check if the target data is considered invalid. If the input features argument is not None, it will be used to check that the target and features have the same dimensions and indices. + + Target data is considered invalid if: + - Target is None. + - Target has NaN or None values. + - Target is of an unsupported Woodwork logical type. + - Target and features have different lengths or indices. + - Target does not have enough instances of a class in a classification problem. + - Target does not contain numeric data for regression problems. + + Args: + X (pd.DataFrame, np.ndarray): Features. If not None, will be used to check that the target and features have the same dimensions and indices. + y (pd.Series, np.ndarray): Target data to check for invalid values. + + Returns: + dict (DataCheckError): List with DataCheckErrors if any invalid values are found in the target data. + + Examples: + >>> import pandas as pd + + Target values must be integers, doubles, or booleans. + + >>> X = pd.DataFrame({"col": [1, 2, 3, 1]}) + >>> y = pd.Series(["cat_1", "cat_2", "cat_1", "cat_2"]) + >>> target_check = InvalidTargetDataCheck("regression", "R2", null_strategy="impute") + >>> assert target_check.validate(X, y) == [ + ... { + ... "message": "Target is unsupported Unknown type. Valid Woodwork logical types include: integer, double, boolean, age, age_fractional, integer_nullable, boolean_nullable, age_nullable", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "error", + ... "details": {"columns": None, "rows": None, "unsupported_type": "unknown"}, + ... "code": "TARGET_UNSUPPORTED_TYPE", + ... "action_options": [] + ... }, + ... { + ... "message": "Target data type should be numeric for regression type problems.", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "error", + ... "details": {"columns": None, "rows": None}, + ... "code": "TARGET_UNSUPPORTED_TYPE_REGRESSION", + ... "action_options": [] + ... } + ... ] + + The target cannot have null values. + + >>> y = pd.Series([None, pd.NA, pd.NaT, None]) + >>> assert target_check.validate(X, y) == [ + ... { + ... "message": "Target is either empty or fully null.", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "error", + ... "details": {"columns": None, "rows": None}, + ... "code": "TARGET_IS_EMPTY_OR_FULLY_NULL", + ... "action_options": [] + ... } + ... ] + ... + ... + >>> y = pd.Series([1, None, 3, None]) + >>> assert target_check.validate(None, y) == [ + ... { + ... "message": "2 row(s) (50.0%) of target values are null", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "error", + ... "details": { + ... "columns": None, + ... "rows": [1, 3], + ... "num_null_rows": 2, + ... "pct_null_rows": 50.0 + ... }, + ... "code": "TARGET_HAS_NULL", + ... "action_options": [ + ... { + ... "code": "IMPUTE_COL", + ... "data_check_name": "InvalidTargetDataCheck", + ... "parameters": { + ... "impute_strategy": { + ... "parameter_type": "global", + ... "type": "category", + ... "categories": ["mean", "most_frequent"], + ... "default_value": "mean" + ... } + ... }, + ... "metadata": {"columns": None, "rows": None, "is_target": True}, + ... } + ... ], + ... } + ... ] + + If the target values don't match the problem type passed, an error will be raised. + In this instance, only two values exist in the target column, but multiclass has been passed as the problem type. + + >>> X = pd.DataFrame([i for i in range(50)]) + >>> y = pd.Series([i%2 for i in range(50)]) + >>> target_check = InvalidTargetDataCheck("multiclass", "Log Loss Multiclass") + >>> assert target_check.validate(X, y) == [ + ... { + ... "message": "Target has two or less classes, which is too few for multiclass problems. Consider changing to binary.", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "error", + ... "details": {"columns": None, "rows": None, "num_classes": 2}, + ... "code": "TARGET_MULTICLASS_NOT_ENOUGH_CLASSES", + ... "action_options": [] + ... } + ... ] + + If the length of X and y differ, a warning will be raised. A warning will also be raised for indices that don"t match. + + >>> target_check = InvalidTargetDataCheck("regression", "R2") + >>> X = pd.DataFrame([i for i in range(5)]) + >>> y = pd.Series([1, 2, 4, 3], index=[1, 2, 4, 3]) + >>> assert target_check.validate(X, y) == [ + ... { + ... "message": "Input target and features have different lengths", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "warning", + ... "details": {"columns": None, "rows": None, "features_length": 5, "target_length": 4}, + ... "code": "MISMATCHED_LENGTHS", + ... "action_options": [] + ... }, + ... { + ... "message": "Input target and features have mismatched indices. Details will include the first 10 mismatched indices.", + ... "data_check_name": "InvalidTargetDataCheck", + ... "level": "warning", + ... "details": { + ... "columns": None, + ... "rows": None, + ... "indices_not_in_features": [], + ... "indices_not_in_target": [0] + ... }, + ... "code": "MISMATCHED_INDICES", + ... "action_options": [] + ... } + ... ] + """ + messages = [] + + if y is None: + messages.append( + DataCheckError( + message="Target is None", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_IS_NONE, + details={}, + ).to_dict(), + ) + return messages + + y = infer_feature_types(y) + messages = self._check_target_has_nan(y, messages) + if any(error["code"] == "TARGET_IS_EMPTY_OR_FULLY_NULL" for error in messages): + # If our target is empty or fully null, no need to check for other invalid targets, return immediately. + return messages + messages = self._check_target_is_unsupported_type(y, messages) + messages = self._check_regression_target(y, messages) + messages = self._check_classification_target(y, messages) + messages = self._check_for_non_positive_target(y, messages) + messages = self._check_target_and_features_compatible(X, y, messages) + return messages + + def _check_target_is_unsupported_type(self, y, messages): + is_supported_type = y.ww.logical_type.type_string in numeric_and_boolean_ww + [ + ww.logical_types.Categorical.type_string, + ] + if not is_supported_type: + messages.append( + DataCheckError( + message="Target is unsupported {} type. Valid Woodwork logical types include: {}".format( + type(y.ww.logical_type), + ", ".join([ltype for ltype in numeric_and_boolean_ww]), + ), + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE, + details={"unsupported_type": y.ww.logical_type.type_string}, + ).to_dict(), + ) + return messages + + def _check_target_has_nan(self, y, messages): + null_rows = y.isnull() + if null_rows.all(): + messages.append( + DataCheckError( + message="Target is either empty or fully null.", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_IS_EMPTY_OR_FULLY_NULL, + details={}, + ).to_dict(), + ) + return messages + elif null_rows.any(): + num_null_rows = null_rows.sum() + pct_null_rows = null_rows.mean() * 100 + rows_to_drop = null_rows.loc[null_rows].index.tolist() + + action_options = [] + impute_action_option = DataCheckActionOption( + DataCheckActionCode.IMPUTE_COL, + data_check_name=self.name, + parameters={ + "impute_strategy": { + "parameter_type": DCAOParameterType.GLOBAL, + "type": "category", + "categories": ["mean", "most_frequent"] + if is_regression(self.problem_type) + else ["most_frequent"], + "default_value": "mean" + if is_regression(self.problem_type) + else "most_frequent", + }, + }, + metadata={"is_target": True}, + ) + drop_action_option = DataCheckActionOption( + DataCheckActionCode.DROP_ROWS, + data_check_name=self.name, + metadata={"is_target": True, "rows": rows_to_drop}, + ) + + if self.null_strategy.lower() == "impute": + action_options.append(impute_action_option) + elif self.null_strategy.lower() == "drop": + action_options.append(drop_action_option) + + messages.append( + DataCheckError( + message="{} row(s) ({}%) of target values are null".format( + num_null_rows, + pct_null_rows, + ), + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_HAS_NULL, + details={ + "num_null_rows": num_null_rows, + "pct_null_rows": pct_null_rows, + "rows": rows_to_drop, + }, + action_options=action_options, + ).to_dict(), + ) + + return messages + + def _check_target_and_features_compatible(self, X, y, messages): + if X is not None: + X = infer_feature_types(X) + X_index = list(X.index) + y_index = list(y.index) + X_length = len(X_index) + y_length = len(y_index) + if X_length != y_length: + messages.append( + DataCheckWarning( + message="Input target and features have different lengths", + data_check_name=self.name, + message_code=DataCheckMessageCode.MISMATCHED_LENGTHS, + details={ + "features_length": X_length, + "target_length": y_length, + }, + ).to_dict(), + ) + + if X_index != y_index: + if set(X_index) == set(y_index): + messages.append( + DataCheckWarning( + message="Input target and features have mismatched indices order.", + data_check_name=self.name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES_ORDER, + details={}, + ).to_dict(), + ) + else: + index_diff_not_in_X = list(set(y_index) - set(X_index))[:10] + index_diff_not_in_y = list(set(X_index) - set(y_index))[:10] + messages.append( + DataCheckWarning( + message="Input target and features have mismatched indices. Details will include the first 10 mismatched indices.", + data_check_name=self.name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES, + details={ + "indices_not_in_features": index_diff_not_in_X, + "indices_not_in_target": index_diff_not_in_y, + }, + ).to_dict(), + ) + return messages + + def _check_for_non_positive_target(self, y, messages): + any_neg = ( + not (y > 0).all() + if y.ww.logical_type.type_string + in [ + ww.logical_types.Integer.type_string, + ww.logical_types.Double.type_string, + ] + else None + ) + if any_neg and self.objective.positive_only: + details = { + "Count of offending values": sum( + val <= 0 for val in y.values.flatten() + ), + } + messages.append( + DataCheckError( + message=f"Target has non-positive values which is not supported for {self.objective.name}", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_INCOMPATIBLE_OBJECTIVE, + details=details, + ).to_dict(), + ) + return messages + + def _check_regression_target(self, y, messages): + if is_regression(self.problem_type) and "numeric" not in y.ww.semantic_tags: + messages.append( + DataCheckError( + message="Target data type should be numeric for regression type problems.", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE_REGRESSION, + details={}, + ).to_dict(), + ) + return messages + + def _check_classification_target(self, y, messages): + value_counts = y.value_counts() + unique_values = value_counts.index.tolist() + + if is_binary(self.problem_type) and len(value_counts) != 2: + if self.n_unique is None: + details = {"target_values": unique_values} + else: + details = { + "target_values": unique_values[ + : min(self.n_unique, len(unique_values)) + ], + } + messages.append( + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details=details, + ).to_dict(), + ) + elif is_multiclass(self.problem_type): + if value_counts.min() <= 1: + least_populated = value_counts[value_counts <= 1] + details = { + "least_populated_class_labels": sorted( + least_populated.index.tolist(), + ), + } + messages.append( + DataCheckError( + message="Target does not have at least two instances per class which is required for multiclass classification", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_TWO_EXAMPLES_PER_CLASS, + details=details, + ).to_dict(), + ) + if len(unique_values) <= 2: + details = {"num_classes": len(unique_values)} + messages.append( + DataCheckError( + message="Target has two or less classes, which is too few for multiclass problems. Consider changing to binary.", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_ENOUGH_CLASSES, + details=details, + ).to_dict(), + ) + + num_class_to_num_value_ratio = len(unique_values) / len(y) + if num_class_to_num_value_ratio >= self.multiclass_continuous_threshold: + details = {"class_to_value_ratio": num_class_to_num_value_ratio} + messages.append( + DataCheckWarning( + message="Target has a large number of unique values, could be regression type problem.", + data_check_name=self.name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_HIGH_UNIQUE_CLASS, + details=details, + ).to_dict(), + ) + return messages \ No newline at end of file diff --git a/checkmates/exceptions/__init__.py b/checkmates/exceptions/__init__.py index 7c36414..5948d61 100644 --- a/checkmates/exceptions/__init__.py +++ b/checkmates/exceptions/__init__.py @@ -3,4 +3,6 @@ DataCheckInitError, MissingComponentError, ValidationErrorCode, + ObjectiveCreationError, + ObjectiveNotFoundError ) diff --git a/checkmates/exceptions/exceptions.py b/checkmates/exceptions/exceptions.py index 8d676b0..0cf3fd2 100644 --- a/checkmates/exceptions/exceptions.py +++ b/checkmates/exceptions/exceptions.py @@ -7,6 +7,13 @@ class MissingComponentError(Exception): pass +class ObjectiveNotFoundError(Exception): + """Exception to raise when specified objective does not exist.""" + + pass + +class ObjectiveCreationError(Exception): + """Exception when get_objective tries to instantiate an objective and required args are not provided.""" class DataCheckInitError(Exception): """Exception raised when a data check can't initialize with the parameters given.""" diff --git a/checkmates/objectives/__init__.py b/checkmates/objectives/__init__.py new file mode 100644 index 0000000..92d200f --- /dev/null +++ b/checkmates/objectives/__init__.py @@ -0,0 +1,11 @@ +from checkmates.objectives.objective_base import ObjectiveBase +from checkmates.objectives.regression_objective import RegressionObjective + +from checkmates.objectives.utils import get_objective +from checkmates.objectives.utils import get_default_primary_search_objective +from checkmates.objectives.utils import get_non_core_objectives +from checkmates.objectives.utils import get_core_objectives + + +from checkmates.objectives.standard_metrics import RootMeanSquaredLogError +from checkmates.objectives.standard_metrics import MeanSquaredLogError \ No newline at end of file diff --git a/checkmates/objectives/objective_base.py b/checkmates/objectives/objective_base.py new file mode 100644 index 0000000..a297eb8 --- /dev/null +++ b/checkmates/objectives/objective_base.py @@ -0,0 +1,217 @@ +"""Base class for all objectives.""" +from abc import ABC, abstractmethod + +import numpy as np +import pandas as pd + +from checkmates.problem_types import handle_problem_types +from checkmates.utils import classproperty + + +class ObjectiveBase(ABC): + """Base class for all objectives.""" + + problem_types = None + + @property + @classmethod + @abstractmethod + def name(cls): + """Returns a name describing the objective.""" + + @property + @classmethod + @abstractmethod + def greater_is_better(cls): + """Returns a boolean determining if a greater score indicates better model performance.""" + + @property + @classmethod + @abstractmethod + def score_needs_proba(cls): + """Returns a boolean determining if the score() method needs probability estimates. + + This should be true for objectives which work with predicted + probabilities, like log loss or AUC, and false for objectives + which compare predicted class labels to the actual labels, like + F1 or correlation. + """ + + @property + @classmethod + @abstractmethod + def perfect_score(cls): + """Returns the score obtained by evaluating this objective on a perfect model.""" + + @property + @classmethod + @abstractmethod + def is_bounded_like_percentage(cls): + """Returns whether this objective is bounded between 0 and 1, inclusive.""" + + @property + @classmethod + @abstractmethod + def expected_range(cls): + """Returns the expected range of the objective, which is not necessarily the possible ranges. + + For example, our expected R2 range is from [-1, 1], although the + actual range is (-inf, 1]. + """ + + @classmethod + @abstractmethod + def objective_function( + cls, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Computes the relative value of the provided predictions compared to the actual labels, according a specified metric. + + Args: + y_predicted (pd.Series): Predicted values of length [n_samples] + y_true (pd.Series): Actual class labels of length [n_samples] + y_train (pd.Series): Observed training values of length [n_samples] + X (pd.DataFrame or np.ndarray): Extra data of shape [n_samples, n_features] necessary to calculate score + sample_weight (pd.DataFrame or np.ndarray): Sample weights used in computing objective value result + + Returns: + Numerical value used to calculate score + """ + + @classproperty + def positive_only(cls): + """If True, this objective is only valid for positive data. Defaults to False.""" + return False + + def score(self, y_true, y_predicted, y_train=None, X=None, sample_weight=None): + """Returns a numerical score indicating performance based on the differences between the predicted and actual values. + + Args: + y_predicted (pd.Series): Predicted values of length [n_samples] + y_true (pd.Series): Actual class labels of length [n_samples] + y_train (pd.Series): Observed training values of length [n_samples] + X (pd.DataFrame or np.ndarray): Extra data of shape [n_samples, n_features] necessary to calculate score + sample_weight (pd.DataFrame or np.ndarray): Sample weights used in computing objective value result + + Returns: + score + """ + if X is not None: + X = self._standardize_input_type(X) + if y_train is not None: + y_train = self._standardize_input_type(y_train) + y_true = self._standardize_input_type(y_true) + y_predicted = self._standardize_input_type(y_predicted) + self.validate_inputs(y_true, y_predicted) + return self.objective_function( + y_true, + y_predicted, + y_train=y_train, + X=X, + sample_weight=sample_weight, + ) + + @staticmethod + def _standardize_input_type(input_data): + """Standardize input to pandas for scoring. + + Args: + input_data (list, pd.DataFrame, pd.Series, or np.ndarray): A matrix of predictions or predicted probabilities + + Returns: + pd.DataFrame or pd.Series: a pd.Series, or pd.DataFrame object if predicted probabilities were provided. + """ + if isinstance(input_data, (pd.Series, pd.DataFrame)): + return input_data + if isinstance(input_data, list): + if isinstance(input_data[0], list): + return pd.DataFrame(input_data) + return pd.Series(input_data) + if isinstance(input_data, np.ndarray): + if len(input_data.shape) == 1: + return pd.Series(input_data) + return pd.DataFrame(input_data) + + def validate_inputs(self, y_true, y_predicted): + """Validates the input based on a few simple checks. + + Args: + y_predicted (pd.Series, or pd.DataFrame): Predicted values of length [n_samples]. + y_true (pd.Series): Actual class labels of length [n_samples]. + + Raises: + ValueError: If the inputs are malformed. + """ + if y_predicted.shape[0] != y_true.shape[0]: + raise ValueError( + "Inputs have mismatched dimensions: y_predicted has shape {}, y_true has shape {}".format( + len(y_predicted), + len(y_true), + ), + ) + if len(y_true) == 0: + raise ValueError("Length of inputs is 0") + + if isinstance(y_true, pd.DataFrame): + y_true = y_true.to_numpy().flatten() + if np.isnan(y_true).any() or np.isinf(y_true).any(): + raise ValueError("y_true contains NaN or infinity") + + if isinstance(y_predicted, pd.DataFrame): + y_predicted = y_predicted.to_numpy().flatten() + if np.isnan(y_predicted).any() or np.isinf(y_predicted).any(): + raise ValueError("y_predicted contains NaN or infinity") + if self.score_needs_proba and np.any([(y_predicted < 0) | (y_predicted > 1)]): + raise ValueError( + "y_predicted contains probability estimates not within [0, 1]", + ) + + @classmethod + def calculate_percent_difference(cls, score, baseline_score): + """Calculate the percent difference between scores. + + Args: + score (float): A score. Output of the score method of this objective. + baseline_score (float): A score. Output of the score method of this objective. In practice, + this is the score achieved on this objective with a baseline estimator. + + Returns: + float: The percent difference between the scores. Note that for objectives that can be interpreted + as percentages, this will be the difference between the reference score and score. For all other + objectives, the difference will be normalized by the reference score. + """ + if pd.isna(score) or pd.isna(baseline_score): + return np.nan + + if np.isclose(baseline_score - score, 0, atol=1e-10): + return 0 + + # Return inf when dividing by 0 + if ( + np.isclose(baseline_score, 0, atol=1e-10) + and not cls.is_bounded_like_percentage + ): + return np.inf + + decrease = False + if (baseline_score > score and cls.greater_is_better) or ( + baseline_score < score and not cls.greater_is_better + ): + decrease = True + + difference = baseline_score - score + change = ( + difference + if cls.is_bounded_like_percentage + else difference / baseline_score + ) + return 100 * (-1) ** (decrease) * np.abs(change) + + @classmethod + def is_defined_for_problem_type(cls, problem_type): + """Returns whether or not an objective is defined for a problem type.""" + return handle_problem_types(problem_type) in cls.problem_types \ No newline at end of file diff --git a/checkmates/objectives/regression_objective.py b/checkmates/objectives/regression_objective.py new file mode 100644 index 0000000..ff7e78e --- /dev/null +++ b/checkmates/objectives/regression_objective.py @@ -0,0 +1,10 @@ +"""Base class for all regression objectives.""" +from checkmates.objectives.objective_base import ObjectiveBase +from checkmates.problem_types import ProblemTypes + + +class RegressionObjective(ObjectiveBase): + """Base class for all regression objectives.""" + + problem_types = [ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION] + """[ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]""" \ No newline at end of file diff --git a/checkmates/objectives/standard_metrics.py b/checkmates/objectives/standard_metrics.py new file mode 100644 index 0000000..fe72e3c --- /dev/null +++ b/checkmates/objectives/standard_metrics.py @@ -0,0 +1,100 @@ +"""Standard machine learning objective functions.""" +import numpy as np +import pandas as pd +from sklearn import metrics + +from checkmates.objectives.regression_objective import RegressionObjective +from checkmates.utils import classproperty + +class RootMeanSquaredLogError(RegressionObjective): + """Root mean squared log error for regression. + + Only valid for nonnegative inputs. Otherwise, will throw a ValueError. + + Example: + >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2]) + >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75]) + >>> np.testing.assert_almost_equal(RootMeanSquaredLogError().objective_function(y_true, y_pred), 0.13090204) + """ + + name = "Root Mean Squared Log Error" + greater_is_better = False + score_needs_proba = False + perfect_score = 0.0 + is_bounded_like_percentage = False # Range [0, Inf) + expected_range = [0, float("inf")] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for root mean squared log error for regression.""" + + def rmsle(y_true, y_pred): + return np.sqrt( + metrics.mean_squared_log_error( + y_true, + y_pred, + sample_weight=sample_weight, + ), + ) + + # Multiseries time series regression + if isinstance(y_true, pd.DataFrame): + raw_rmsles = [] + for i in range(len(y_true.columns)): + y_true_i = y_true.iloc[:, i] + y_predicted_i = y_predicted.iloc[:, i] + raw_rmsles.append(rmsle(y_true_i, y_predicted_i)) + return np.mean(raw_rmsles) + + # All univariate regression + return rmsle(y_true, y_predicted) + + @classproperty + def positive_only(self): + """If True, this objective is only valid for positive data.""" + return True + + +class MeanSquaredLogError(RegressionObjective): + """Mean squared log error for regression. + + Only valid for nonnegative inputs. Otherwise, will throw a ValueError. + + Example: + >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2]) + >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75]) + >>> np.testing.assert_almost_equal(MeanSquaredLogError().objective_function(y_true, y_pred), 0.0171353) + """ + + name = "Mean Squared Log Error" + greater_is_better = False + score_needs_proba = False + perfect_score = 0.0 + is_bounded_like_percentage = False # Range [0, Inf) + expected_range = [0, float("inf")] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for mean squared log error for regression.""" + return metrics.mean_squared_log_error( + y_true, + y_predicted, + sample_weight=sample_weight, + ) + + @classproperty + def positive_only(self): + """If True, this objective is only valid for positive data.""" + return True \ No newline at end of file diff --git a/checkmates/objectives/utils.py b/checkmates/objectives/utils.py new file mode 100644 index 0000000..0759169 --- /dev/null +++ b/checkmates/objectives/utils.py @@ -0,0 +1,139 @@ +"""Utility methods for EvalML objectives.""" +from checkmates import objectives +from checkmates.exceptions import ObjectiveCreationError, ObjectiveNotFoundError +from checkmates.objectives.objective_base import ObjectiveBase +from checkmates.utils.gen_utils import _get_subclasses +from checkmates.problem_types import handle_problem_types + +def get_non_core_objectives(): + """Get non-core objective classes. + + Non-core objectives are objectives that are domain-specific. Users typically need to configure these objectives + before using them in AutoMLSearch. + + Returns: + List of ObjectiveBase classes + """ + return [ + objectives.MeanSquaredLogError, + objectives.RootMeanSquaredLogError, + ] + +def _all_objectives_dict(): + all_objectives = _get_subclasses(ObjectiveBase) + objectives_dict = {} + for objective in all_objectives: + if "evalml.objectives" not in objective.__module__: + continue + objectives_dict[objective.name.lower()] = objective + return objectives_dict + +def get_objective(objective, return_instance=False, **kwargs): + """Returns the Objective class corresponding to a given objective name. + + Args: + objective (str or ObjectiveBase): Name or instance of the objective class. + return_instance (bool): Whether to return an instance of the objective. This only applies if objective + is of type str. Note that the instance will be initialized with default arguments. + kwargs (Any): Any keyword arguments to pass into the objective. Only used when return_instance=True. + + Returns: + ObjectiveBase if the parameter objective is of type ObjectiveBase. If objective is instead a valid + objective name, function will return the class corresponding to that name. If return_instance is True, + an instance of that objective will be returned. + + Raises: + TypeError: If objective is None. + TypeError: If objective is not a string and not an instance of ObjectiveBase. + ObjectiveNotFoundError: If input objective is not a valid objective. + ObjectiveCreationError: If objective cannot be created properly. + """ + if objective is None: + raise TypeError("Objective parameter cannot be NoneType") + if isinstance(objective, ObjectiveBase): + return objective + all_objectives_dict = _all_objectives_dict() + if not isinstance(objective, str): + raise TypeError( + "If parameter objective is not a string, it must be an instance of ObjectiveBase!", + ) + if objective.lower() not in all_objectives_dict: + raise ObjectiveNotFoundError( + f"{objective} is not a valid Objective! " + "Use evalml.objectives.get_all_objective_names() " + "to get a list of all valid objective names. ", + ) + + objective_class = all_objectives_dict[objective.lower()] + + if return_instance: + try: + return objective_class(**kwargs) + except TypeError as e: + raise ObjectiveCreationError( + f"In get_objective, cannot pass in return_instance=True for {objective} because {str(e)}", + ) + + return objective_class + +def get_default_primary_search_objective(problem_type): + """Get the default primary search objective for a problem type. + + Args: + problem_type (str or ProblemType): Problem type of interest. + + Returns: + ObjectiveBase: primary objective instance for the problem type. + """ + problem_type = handle_problem_types(problem_type) + objective_name = { + "binary": "Log Loss Binary", + "multiclass": "Log Loss Multiclass", + "regression": "R2", + "time series regression": "MedianAE", + "time series binary": "Log Loss Binary", + "time series multiclass": "Log Loss Multiclass", + }[problem_type.value] + return get_objective(objective_name, return_instance=True) + +def get_core_objectives(problem_type): + """Returns all core objective instances associated with the given problem type. + + Core objectives are designed to work out-of-the-box for any dataset. + + Args: + problem_type (str/ProblemTypes): Type of problem + + Returns: + List of ObjectiveBase instances + + Examples: + >>> for objective in get_core_objectives("regression"): + ... print(objective.name) + ExpVariance + MaxError + MedianAE + MSE + MAE + R2 + Root Mean Squared Error + >>> for objective in get_core_objectives("binary"): + ... print(objective.name) + MCC Binary + Log Loss Binary + Gini + AUC + Precision + F1 + Balanced Accuracy Binary + Accuracy Binary + """ + problem_type = handle_problem_types(problem_type) + all_objectives_dict = _all_objectives_dict() + objectives = [ + obj() + for obj in all_objectives_dict.values() + if obj.is_defined_for_problem_type(problem_type) + and obj not in get_non_core_objectives() + ] + return objectives diff --git a/checkmates/utils/gen_utils.py b/checkmates/utils/gen_utils.py index e4ac4e7..e8e2b19 100644 --- a/checkmates/utils/gen_utils.py +++ b/checkmates/utils/gen_utils.py @@ -4,6 +4,29 @@ logger = logging.getLogger(__name__) +def _get_subclasses(base_class): + """Gets all of the leaf nodes in the hiearchy tree for a given base class. + + Args: + base_class (abc.ABCMeta): Class to find all of the children for. + + Returns: + subclasses (list): List of all children that are not base classes. + """ + classes_to_check = base_class.__subclasses__() + subclasses = [] + + while classes_to_check: + subclass = classes_to_check.pop() + children = subclass.__subclasses__() + + if children: + classes_to_check.extend(children) + else: + subclasses.append(subclass) + + return subclasses + class classproperty: """Allows function to be accessed as a class level property. diff --git a/tests/conftest.py b/tests/conftest.py index 4ddf157..6fccbb8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,8 @@ import woodwork as ww from woodwork import logical_types as ww_logical_types from sklearn import datasets +from checkmates.objectives.utils import get_core_objectives +from checkmates.problem_types import ProblemTypes def pytest_configure(config): @@ -128,6 +130,10 @@ def X_y_regression(): y = ww.init_series(pd.Series(y), logical_type="double") return X, y +@pytest.fixture +def time_series_core_objectives(): + return get_core_objectives(ProblemTypes.TIME_SERIES_REGRESSION) + @pytest.fixture def make_data_type(): """Helper function to convert numpy or pandas input to the appropriate type for tests.""" diff --git a/tests/data_checks_tests/test_invalid_target_data_check.py b/tests/data_checks_tests/test_invalid_target_data_check.py new file mode 100644 index 0000000..2cadeda --- /dev/null +++ b/tests/data_checks_tests/test_invalid_target_data_check.py @@ -0,0 +1,820 @@ +import numpy as np +import pandas as pd +import pytest +import woodwork as ww + +from checkmates.objectives.utils import get_default_primary_search_objective +from checkmates.data_checks import ( + DataCheckActionCode, + DataCheckActionOption, + DataCheckError, + DataCheckMessageCode, + DataChecks, + DataCheckWarning, + DCAOParameterType, + InvalidTargetDataCheck, +) +from checkmates.exceptions import DataCheckInitError +from checkmates.objectives import ( + MeanSquaredLogError, + RootMeanSquaredLogError, +) +from checkmates.problem_types import ProblemTypes, is_binary, is_multiclass, is_regression +from checkmates.utils.woodwork_utils import numeric_and_boolean_ww + +invalid_targets_data_check_name = InvalidTargetDataCheck.name + + +def test_invalid_target_data_check_invalid_n_unique(): + with pytest.raises( + ValueError, + match="`n_unique` must be a non-negative integer value.", + ): + InvalidTargetDataCheck( + "regression", + get_default_primary_search_objective("regression"), + n_unique=-1, + ) + + +@pytest.mark.parametrize("null_strategy", ["invalid", None]) +def test_invalid_target_data_check_invalid_null_strategy(null_strategy): + with pytest.raises( + ValueError, + match="The acceptable values for 'null_strategy' are 'impute' and 'drop'.", + ): + InvalidTargetDataCheck( + "regression", + get_default_primary_search_objective("regression"), + null_strategy=null_strategy, + ) + + +def test_invalid_target_data_check_nan_error(): + X = pd.DataFrame({"col": [1, 2, 3]}) + invalid_targets_check = InvalidTargetDataCheck( + "regression", + get_default_primary_search_objective("regression"), + ) + + assert invalid_targets_check.validate(X, y=pd.Series([1, 2, 3])) == [] + assert invalid_targets_check.validate(X, y=pd.Series([np.nan, np.nan, np.nan])) == [ + DataCheckError( + message="Target is either empty or fully null.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_IS_EMPTY_OR_FULLY_NULL, + details={}, + ).to_dict(), + ] + + +def test_invalid_target_data_check_numeric_binary_classification_valid_float(): + y = pd.Series([0.0, 1.0, 0.0, 1.0]) + X = pd.DataFrame({"col": range(len(y))}) + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + ) + assert invalid_targets_check.validate(X, y) == [] + + +def test_invalid_target_data_check_multiclass_two_examples_per_class(): + y = pd.Series([0] + [1] * 19 + [2] * 80) + X = pd.DataFrame({"col": range(len(y))}) + invalid_targets_check = InvalidTargetDataCheck( + "multiclass", + get_default_primary_search_objective("binary"), + ) + expected_message = "Target does not have at least two instances per class which is required for multiclass classification" + + # with 1 class not having min 2 instances + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message=expected_message, + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_TWO_EXAMPLES_PER_CLASS, + details={"least_populated_class_labels": [0]}, + ).to_dict(), + ] + + y = pd.Series([0] + [1] + [2] * 98) + X = pd.DataFrame({"col": range(len(y))}) + # with 2 classes not having min 2 instances + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message=expected_message, + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_TWO_EXAMPLES_PER_CLASS, + details={"least_populated_class_labels": [0, 1]}, + ).to_dict(), + ] + + +@pytest.mark.parametrize( + "pd_type", + ["int16", "int32", "int64", "float16", "float32", "float64", "bool"], +) +def test_invalid_target_data_check_invalid_pandas_data_types_error(pd_type): + y = pd.Series([0, 1, 0, 0, 1, 0, 1, 0]) + y = y.astype(pd_type) + X = pd.DataFrame({"col": range(len(y))}) + + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + ) + + assert invalid_targets_check.validate(X, y) == [] + + y = pd.Series(pd.date_range("2000-02-03", periods=5, freq="W")) + X = pd.DataFrame({"col": range(len(y))}) + + unique_values = y.value_counts().index.tolist() + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message="Target is unsupported {} type. Valid Woodwork logical types include: {}".format( + "Datetime", + ", ".join([ltype for ltype in numeric_and_boolean_ww]), + ), + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE, + details={"unsupported_type": "datetime"}, + ).to_dict(), + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": unique_values}, + ).to_dict(), + ] + + +def test_invalid_target_y_none(): + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + ) + assert invalid_targets_check.validate(pd.DataFrame(), y=None) == [ + DataCheckError( + message="Target is None", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_IS_NONE, + details={}, + ).to_dict(), + ] + + +@pytest.mark.parametrize("null_strategy", ["Impute", "DROP"]) +def test_invalid_target_data_null_strategies(null_strategy): + invalid_targets_check = InvalidTargetDataCheck( + "regression", + get_default_primary_search_objective("regression"), + null_strategy=null_strategy, + ) + + expected_action_options = [] + impute_action_option = DataCheckActionOption( + DataCheckActionCode.IMPUTE_COL, + data_check_name=invalid_targets_data_check_name, + parameters={ + "impute_strategy": { + "parameter_type": DCAOParameterType.GLOBAL, + "type": "category", + "categories": ["mean", "most_frequent"], + "default_value": "mean", + }, + }, + metadata={"is_target": True}, + ) + drop_action_option = DataCheckActionOption( + DataCheckActionCode.DROP_ROWS, + data_check_name=invalid_targets_data_check_name, + metadata={"is_target": True, "rows": [0, 3]}, + ) + if null_strategy.lower() == "impute": + expected_action_options.append(impute_action_option) + elif null_strategy.lower() == "drop": + expected_action_options.append(drop_action_option) + + expected = [ + DataCheckError( + message="2 row(s) (40.0%) of target values are null", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_HAS_NULL, + details={"num_null_rows": 2, "pct_null_rows": 40.0, "rows": [0, 3]}, + action_options=expected_action_options, + ).to_dict(), + ] + + y = pd.Series([None, 3.5, 2.8, None, 0]) + X = pd.DataFrame({"col": range(len(y))}) + + messages = invalid_targets_check.validate(X, y) + assert messages == expected + + +def test_invalid_target_data_input_formats(): + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + null_strategy="impute", + ) + + # test empty pd.Series + X = pd.DataFrame() + messages = invalid_targets_check.validate(X, pd.Series()) + assert messages == [ + DataCheckError( + message="Target is either empty or fully null.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_IS_EMPTY_OR_FULLY_NULL, + details={}, + ).to_dict(), + ] + + expected = [ + DataCheckError( + message="3 row(s) (75.0%) of target values are null", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_HAS_NULL, + details={"num_null_rows": 3, "pct_null_rows": 75, "rows": [0, 1, 2]}, + action_options=[ + DataCheckActionOption( + DataCheckActionCode.IMPUTE_COL, + data_check_name=invalid_targets_data_check_name, + parameters={ + "impute_strategy": { + "parameter_type": DCAOParameterType.GLOBAL, + "type": "category", + "categories": ["most_frequent"], + "default_value": "most_frequent", + }, + }, + metadata={"is_target": True}, + ), + ], + ).to_dict(), + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": [0]}, + ).to_dict(), + ] + # test Woodwork + y = pd.Series([None, None, None, 0]) + X = pd.DataFrame({"col": range(len(y))}) + + messages = invalid_targets_check.validate(X, y) + assert messages == expected + + # test list + y = [np.nan, np.nan, np.nan, 0] + X = pd.DataFrame({"col": range(len(y))}) + + messages = invalid_targets_check.validate(X, y) + assert messages == expected + + # test np.array + y = np.array([np.nan, np.nan, np.nan, 0]) + X = pd.DataFrame({"col": range(len(y))}) + + messages = invalid_targets_check.validate(X, y) + assert messages == expected + + +@pytest.mark.parametrize( + "problem_type", + [ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY], +) +def test_invalid_target_data_check_n_unique(problem_type): + y = pd.Series(list(range(100, 200)) + list(range(200))) + unique_values = y.value_counts().index.tolist()[:100] # n_unique defaults to 100 + X = pd.DataFrame({"col": range(len(y))}) + + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + ) + # Test default value of n_unique + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": unique_values}, + ).to_dict(), + ] + + # Test number of unique values < n_unique + y = pd.Series(range(20)) + X = pd.DataFrame({"col": range(len(y))}) + + unique_values = y.value_counts().index.tolist() + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": unique_values}, + ).to_dict(), + ] + + # Test n_unique is None + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + n_unique=None, + ) + y = pd.Series(range(150)) + X = pd.DataFrame({"col": range(len(y))}) + + unique_values = y.value_counts().index.tolist() + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": unique_values}, + ).to_dict(), + ] + + +@pytest.mark.parametrize( + "objective", + [ + "Root Mean Squared Log Error", + "Mean Squared Log Error", + ], +) +def test_invalid_target_data_check_invalid_labels_for_nonnegative_objective_names( + objective, +): + X = pd.DataFrame({"column_one": [100, 200, 100, 200, 200, 100, 200, 100] * 25}) + y = pd.Series([2, 2, 3, 3, -1, -1, 1, 1] * 25) + + data_checks = DataChecks( + [InvalidTargetDataCheck], + { + "InvalidTargetDataCheck": { + "problem_type": "multiclass", + "objective": objective, + }, + }, + ) + assert data_checks.validate(X, y) == [ + DataCheckError( + message=f"Target has non-positive values which is not supported for {objective}", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_INCOMPATIBLE_OBJECTIVE, + details={ + "Count of offending values": sum( + val <= 0 for val in y.values.flatten() + ), + }, + ).to_dict(), + ] + + X = pd.DataFrame({"column_one": [100, 200, 100, 200, 100]}) + y = pd.Series([2, 3, 0, 1, 1]) + + invalid_targets_check = InvalidTargetDataCheck( + problem_type="regression", + objective=objective, + ) + + assert invalid_targets_check.validate(X, y) == [ + DataCheckError( + message=f"Target has non-positive values which is not supported for {objective}", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_INCOMPATIBLE_OBJECTIVE, + details={ + "Count of offending values": sum( + val <= 0 for val in y.values.flatten() + ), + }, + ).to_dict(), + ] + + +@pytest.mark.parametrize( + "objective", + [RootMeanSquaredLogError(), MeanSquaredLogError()], +) +def test_invalid_target_data_check_invalid_labels_for_nonnegative_objective_instances( + objective, +): + X = pd.DataFrame({"column_one": [100, 200, 100, 200, 200, 100, 200, 100] * 25}) + y = pd.Series([2, 2, 3, 3, -1, -1, 1, 1] * 25) + + data_checks = DataChecks( + [InvalidTargetDataCheck], + { + "InvalidTargetDataCheck": { + "problem_type": "multiclass", + "objective": objective, + }, + }, + ) + + assert data_checks.validate(X, y) == [ + DataCheckError( + message=f"Target has non-positive values which is not supported for {objective.name}", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_INCOMPATIBLE_OBJECTIVE, + details={ + "Count of offending values": sum( + val <= 0 for val in y.values.flatten() + ), + }, + ).to_dict(), + ] + + +def test_invalid_target_data_check_invalid_labels_for_objectives( + time_series_core_objectives, +): + X = pd.DataFrame({"column_one": [100, 200, 100, 200, 200, 100, 200, 100] * 25}) + y = pd.Series([2, 2, 3, 3, -1, -1, 1, 1] * 25) + + for objective in time_series_core_objectives: + if not objective.positive_only: + data_checks = DataChecks( + [InvalidTargetDataCheck], + { + "InvalidTargetDataCheck": { + "problem_type": "multiclass", + "objective": objective, + }, + }, + ) + assert data_checks.validate(X, y) == [] + + X = pd.DataFrame({"column_one": [100, 200, 100, 200, 100]}) + y = pd.Series([2, 3, 0, 1, 1]) + + for objective in time_series_core_objectives: + if not objective.positive_only: + invalid_targets_check = InvalidTargetDataCheck( + problem_type="regression", + objective=objective, + ) + assert invalid_targets_check.validate(X, y) == [] + + +@pytest.mark.parametrize( + "objective", + [ + "Root Mean Squared Log Error", + "Mean Squared Log Error", + ], +) +def test_invalid_target_data_check_valid_labels_for_nonnegative_objectives(objective): + X = pd.DataFrame({"column_one": [100, 100, 200, 300, 100, 200, 100] * 25}) + y = pd.Series([2, 2, 3, 3, 1, 1, 1] * 25) + + data_checks = DataChecks( + [InvalidTargetDataCheck], + { + "InvalidTargetDataCheck": { + "problem_type": "multiclass", + "objective": objective, + }, + }, + ) + assert data_checks.validate(X, y) == [] + + +def test_invalid_target_data_check_initialize_with_none_objective(): + with pytest.raises(DataCheckInitError, match="Encountered the following error"): + DataChecks( + [InvalidTargetDataCheck], + { + "InvalidTargetDataCheck": { + "problem_type": "multiclass", + "objective": None, + }, + }, + ) + + +@pytest.mark.parametrize( + "problem_type", + [ProblemTypes.TIME_SERIES_REGRESSION, ProblemTypes.REGRESSION], +) +def test_invalid_target_data_check_regression_problem_nonnumeric_data(problem_type): + y_categorical = pd.Series(["Peace", "Is", "A", "Lie"] * 100) + y_mixed_cat_numeric = pd.Series(["Peace", 2, "A", 4] * 100) + y_integer = pd.Series([1, 2, 3, 4]) + y_float = pd.Series([1.1, 2.2, 3.3, 4.4]) + y_numeric = pd.Series([1, 2.2, 3, 4.4]) + + data_check_error = DataCheckError( + message="Target data type should be numeric for regression type problems.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE_REGRESSION, + details={}, + ).to_dict() + + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + ) + assert invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_categorical))}), + y=y_categorical, + ) == [data_check_error] + assert invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_mixed_cat_numeric))}), + y=y_mixed_cat_numeric, + ) == [data_check_error] + assert ( + invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_integer))}), + y=y_integer, + ) + == [] + ) + assert ( + invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_float))}), + y=y_float, + ) + == [] + ) + assert ( + invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_numeric))}), + y=y_numeric, + ) + == [] + ) + + +@pytest.mark.parametrize( + "problem_type", + [ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS], +) +def test_invalid_target_data_check_multiclass_problem_binary_data(problem_type): + y_multiclass = pd.Series([1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3] * 25) + y_binary = pd.Series([0, 1, 1, 1, 0, 0] * 25) + + data_check_error = DataCheckError( + message="Target has two or less classes, which is too few for multiclass problems. Consider changing to binary.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_ENOUGH_CLASSES, + details={"num_classes": len(set(y_binary))}, + ).to_dict() + + invalid_targets_check = InvalidTargetDataCheck( + "multiclass", + get_default_primary_search_objective("multiclass"), + ) + assert ( + invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_multiclass))}), + y=y_multiclass, + ) + == [] + ) + assert invalid_targets_check.validate( + X=pd.DataFrame({"col": range(len(y_binary))}), + y=y_binary, + ) == [data_check_error] + + +@pytest.mark.parametrize( + "problem_type", + [ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS], +) +def test_invalid_target_data_check_multiclass_problem_almost_continuous_data( + problem_type, +): + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + ) + y_multiclass_high_classes = pd.Series( + list(range(0, 100)) * 3, + ) # 100 classes, 300 samples, .33 class/sample ratio + X = pd.DataFrame({"col": range(len(y_multiclass_high_classes))}) + data_check_warning = DataCheckWarning( + message="Target has a large number of unique values, could be regression type problem.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_HIGH_UNIQUE_CLASS, + details={"class_to_value_ratio": 1 / 3}, + ).to_dict() + assert invalid_targets_check.validate(X, y=y_multiclass_high_classes) == [ + data_check_warning, + ] + + y_multiclass_med_classes = pd.Series( + list(range(0, 5)) * 20, + ) # 5 classes, 100 samples, .05 class/sample ratio + X = pd.DataFrame({"col": range(len(y_multiclass_med_classes))}) + data_check_warning = DataCheckWarning( + message="Target has a large number of unique values, could be regression type problem.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_HIGH_UNIQUE_CLASS, + details={"class_to_value_ratio": 0.05}, + ).to_dict() + assert invalid_targets_check.validate(X, y=y_multiclass_med_classes) == [ + data_check_warning, + ] + + y_multiclass_low_classes = pd.Series( + list(range(0, 3)) * 100, + ) # 2 classes, 300 samples, .01 class/sample ratio + X = pd.DataFrame({"col": range(len(y_multiclass_low_classes))}) + assert invalid_targets_check.validate(X, y=y_multiclass_low_classes) == [] + + +def test_invalid_target_data_check_mismatched_indices(): + X = pd.DataFrame({"col": [1, 2, 3]}) + y_same_index = pd.Series([1, 0, 1]) + y_diff_index = pd.Series([0, 1, 0], index=[1, 5, 10]) + y_diff_index_order = pd.Series([0, 1, 0], index=[0, 2, 1]) + + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + ) + assert invalid_targets_check.validate(X=None, y=y_same_index) == [] + assert invalid_targets_check.validate(X, y_same_index) == [] + + X_index_missing = list(set(y_diff_index.index) - set(X.index)) + y_index_missing = list(set(X.index) - set(y_diff_index.index)) + assert invalid_targets_check.validate(X, y_diff_index) == [ + DataCheckWarning( + message="Input target and features have mismatched indices. Details will include the first 10 mismatched indices.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES, + details={ + "indices_not_in_features": X_index_missing, + "indices_not_in_target": y_index_missing, + }, + ).to_dict(), + ] + assert invalid_targets_check.validate(X, y_diff_index_order) == [ + DataCheckWarning( + message="Input target and features have mismatched indices order.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES_ORDER, + details={}, + ).to_dict(), + ] + + # Test that we only store ten mismatches when there are more than 10 differences in indices found + X_large = pd.DataFrame({"col": range(20)}) + y_more_than_ten_diff_indices = pd.Series([0, 1] * 10, index=range(20, 40)) + X_index_missing = list(set(y_more_than_ten_diff_indices.index) - set(X.index)) + y_index_missing = list(set(X_large.index) - set(y_more_than_ten_diff_indices.index)) + assert invalid_targets_check.validate(X_large, y_more_than_ten_diff_indices) == [ + DataCheckWarning( + message="Input target and features have mismatched indices. Details will include the first 10 mismatched indices.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES, + details={ + "indices_not_in_features": X_index_missing[:10], + "indices_not_in_target": y_index_missing[:10], + }, + ).to_dict(), + ] + + +def test_invalid_target_data_check_different_lengths(): + X = pd.DataFrame({"col": [1, 2, 3]}) + y_diff_len = pd.Series([0, 1]) + invalid_targets_check = InvalidTargetDataCheck( + "binary", + get_default_primary_search_objective("binary"), + ) + assert invalid_targets_check.validate(X, y_diff_len) == [ + DataCheckWarning( + message="Input target and features have different lengths", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.MISMATCHED_LENGTHS, + details={ + "features_length": len(X.index), + "target_length": len(y_diff_len.index), + }, + ).to_dict(), + DataCheckWarning( + message="Input target and features have mismatched indices. Details will include the first 10 mismatched indices.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.MISMATCHED_INDICES, + details={"indices_not_in_features": [], "indices_not_in_target": [2]}, + ).to_dict(), + ] + + +@pytest.mark.parametrize( + "problem_type", + [ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY], +) +def test_invalid_target_data_check_numeric_binary_does_not_return_warnings( + problem_type, +): + y = pd.Series([1, 5, 1, 5, 1, 1]) + X = pd.DataFrame({"col": range(len(y))}) + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + ) + assert invalid_targets_check.validate(X, y) == [] + + +@pytest.mark.parametrize("use_nullable_types", [True, False]) +@pytest.mark.parametrize("problem_type", ProblemTypes.all_problem_types) +def test_invalid_target_data_check_action_for_data_with_null( + use_nullable_types, + problem_type, +): + y = pd.Series([None, None, None, 0, 0, 0, 0, 0, 0, 0]) + if use_nullable_types: + y = ww.init_series(y, logical_type="IntegerNullable") + + X = pd.DataFrame({"col": range(len(y))}) + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + null_strategy="impute", + ) + expected = [ + DataCheckError( + message="3 row(s) (30.0%) of target values are null", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_HAS_NULL, + details={"num_null_rows": 3, "pct_null_rows": 30.0, "rows": [0, 1, 2]}, + action_options=[ + DataCheckActionOption( + DataCheckActionCode.IMPUTE_COL, + data_check_name=invalid_targets_data_check_name, + parameters={ + "impute_strategy": { + "parameter_type": DCAOParameterType.GLOBAL, + "type": "category", + "categories": ["mean", "most_frequent"] + if is_regression(problem_type) + else ["most_frequent"], + "default_value": "mean" + if is_regression(problem_type) + else "most_frequent", + }, + }, + metadata={"is_target": True}, + ), + ], + ).to_dict(), + ] + if is_binary(problem_type): + expected.append( + DataCheckError( + message="Binary class targets require exactly two unique values.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_BINARY_NOT_TWO_UNIQUE_VALUES, + details={"target_values": [0]}, + ).to_dict(), + ) + elif is_multiclass(problem_type): + expected.append( + DataCheckError( + message="Target has two or less classes, which is too few for multiclass problems. Consider changing to binary.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_NOT_ENOUGH_CLASSES, + details={"num_classes": 1}, + ).to_dict(), + ) + expected.append( + DataCheckWarning( + message="Target has a large number of unique values, could be regression type problem.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_MULTICLASS_HIGH_UNIQUE_CLASS, + details={"class_to_value_ratio": 0.1}, + ).to_dict(), + ) + + messages = invalid_targets_check.validate(X, y) + assert messages == expected + + +@pytest.mark.parametrize("problem_type", ProblemTypes.all_problem_types) +def test_invalid_target_data_check_action_for_all_null(problem_type): + invalid_targets_check = InvalidTargetDataCheck( + problem_type, + get_default_primary_search_objective(problem_type), + ) + + y_all_null = pd.Series([None, None, None]) + X = pd.DataFrame({"col": range(len(y_all_null))}) + + expected = [ + DataCheckError( + message="Target is either empty or fully null.", + data_check_name=invalid_targets_data_check_name, + message_code=DataCheckMessageCode.TARGET_IS_EMPTY_OR_FULLY_NULL, + details={}, + ).to_dict(), + ] + messages = invalid_targets_check.validate(X, y_all_null) + assert messages == expected \ No newline at end of file From 7145f77ea4f3a1157dff9aee0b8e402f1754dfd8 Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Mon, 14 Aug 2023 12:25:08 -0400 Subject: [PATCH 2/8] release notes updated --- docs/source/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 638ad36..876e510 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -2,6 +2,7 @@ Release Notes ------------- **Future Releases** * Enhancements + * Added all datachecks except `invalid_target_data_check`` along with tests and utils, migrated over from `EvalML` :pr:`15` * Fixes * Changes * Documentation Changes From 23b944ed849087306a328f31967baaab21f47e33 Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Mon, 14 Aug 2023 12:30:02 -0400 Subject: [PATCH 3/8] lint fix --- checkmates/data_checks/__init__.py | 33 ++++++++++++++----- .../checks/class_imbalance_data_check.py | 2 +- .../checks/datetime_format_data_check.py | 2 +- .../checks/invalid_target_data_check.py | 2 +- .../checks/multicollinearity_data_check.py | 2 +- .../checks/no_variance_data_check.py | 2 +- .../data_checks/checks/null_data_check.py | 2 +- .../data_checks/checks/outliers_data_check.py | 2 +- .../data_checks/checks/sparsity_data_check.py | 2 +- .../checks/target_distribution_data_check.py | 2 +- .../checks/target_leakage_data_check.py | 2 +- .../checks/ts_parameters_data_check.py | 2 +- .../checks/ts_splitting_data_check.py | 2 +- .../checks/uniqueness_data_check.py | 2 +- checkmates/exceptions/__init__.py | 2 +- checkmates/exceptions/exceptions.py | 3 ++ checkmates/objectives/__init__.py | 4 ++- checkmates/objectives/objective_base.py | 2 +- checkmates/objectives/regression_objective.py | 2 +- checkmates/objectives/standard_metrics.py | 3 +- checkmates/objectives/utils.py | 7 +++- checkmates/utils/gen_utils.py | 3 ++ tests/conftest.py | 6 +++- .../test_class_imbalance_data_check.py | 2 +- .../test_datetime_format_data_check.py | 2 +- .../test_invalid_target_data_check.py | 11 +++++-- .../test_multicollinearity_data_check.py | 2 +- .../test_no_variance_data_check.py | 2 +- .../data_checks_tests/test_null_data_check.py | 2 +- .../test_outliers_data_check.py | 2 +- .../test_sparsity_data_check.py | 2 +- .../test_target_distribution_data_check.py | 2 +- .../test_target_leakage_data_check.py | 2 +- .../test_ts_parameters_data_check.py | 2 +- .../test_ts_splitting_data_check.py | 2 +- .../test_uniqueness_data_check.py | 2 +- 36 files changed, 82 insertions(+), 44 deletions(-) diff --git a/checkmates/data_checks/__init__.py b/checkmates/data_checks/__init__.py index 29da738..42b3254 100644 --- a/checkmates/data_checks/__init__.py +++ b/checkmates/data_checks/__init__.py @@ -24,19 +24,34 @@ ) from checkmates.data_checks.checks.id_columns_data_check import IDColumnsDataCheck from checkmates.data_checks.checks.null_data_check import NullDataCheck -from checkmates.data_checks.checks.class_imbalance_data_check import ClassImbalanceDataCheck +from checkmates.data_checks.checks.class_imbalance_data_check import ( + ClassImbalanceDataCheck, +) from checkmates.data_checks.checks.no_variance_data_check import NoVarianceDataCheck from checkmates.data_checks.checks.outliers_data_check import OutliersDataCheck from checkmates.data_checks.checks.uniqueness_data_check import UniquenessDataCheck -from checkmates.data_checks.checks.ts_splitting_data_check import TimeSeriesSplittingDataCheck -from checkmates.data_checks.checks.ts_parameters_data_check import TimeSeriesParametersDataCheck -from checkmates.data_checks.checks.target_leakage_data_check import TargetLeakageDataCheck -from checkmates.data_checks.checks.target_distribution_data_check import TargetDistributionDataCheck +from checkmates.data_checks.checks.ts_splitting_data_check import ( + TimeSeriesSplittingDataCheck, +) +from checkmates.data_checks.checks.ts_parameters_data_check import ( + TimeSeriesParametersDataCheck, +) +from checkmates.data_checks.checks.target_leakage_data_check import ( + TargetLeakageDataCheck, +) +from checkmates.data_checks.checks.target_distribution_data_check import ( + TargetDistributionDataCheck, +) from checkmates.data_checks.checks.sparsity_data_check import SparsityDataCheck -from checkmates.data_checks.checks.datetime_format_data_check import DateTimeFormatDataCheck -from checkmates.data_checks.checks.multicollinearity_data_check import MulticollinearityDataCheck -from checkmates.data_checks.checks.invalid_target_data_check import InvalidTargetDataCheck - +from checkmates.data_checks.checks.datetime_format_data_check import ( + DateTimeFormatDataCheck, +) +from checkmates.data_checks.checks.multicollinearity_data_check import ( + MulticollinearityDataCheck, +) +from checkmates.data_checks.checks.invalid_target_data_check import ( + InvalidTargetDataCheck, +) from checkmates.data_checks.datacheck_meta.utils import handle_data_check_action_code diff --git a/checkmates/data_checks/checks/class_imbalance_data_check.py b/checkmates/data_checks/checks/class_imbalance_data_check.py index d07e40e..f8e310a 100644 --- a/checkmates/data_checks/checks/class_imbalance_data_check.py +++ b/checkmates/data_checks/checks/class_imbalance_data_check.py @@ -233,4 +233,4 @@ def validate(self, X, y): details={"target_values": severe_imbalance}, ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/datetime_format_data_check.py b/checkmates/data_checks/checks/datetime_format_data_check.py index 7e01e26..4d20d5e 100644 --- a/checkmates/data_checks/checks/datetime_format_data_check.py +++ b/checkmates/data_checks/checks/datetime_format_data_check.py @@ -484,4 +484,4 @@ def validate(self, X, y): ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/invalid_target_data_check.py b/checkmates/data_checks/checks/invalid_target_data_check.py index fc92625..39b8de4 100644 --- a/checkmates/data_checks/checks/invalid_target_data_check.py +++ b/checkmates/data_checks/checks/invalid_target_data_check.py @@ -445,4 +445,4 @@ def _check_classification_target(self, y, messages): details=details, ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/multicollinearity_data_check.py b/checkmates/data_checks/checks/multicollinearity_data_check.py index 5a3c27a..bbed8d3 100644 --- a/checkmates/data_checks/checks/multicollinearity_data_check.py +++ b/checkmates/data_checks/checks/multicollinearity_data_check.py @@ -72,4 +72,4 @@ def validate(self, X, y=None): details={"columns": correlated_cols}, ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/no_variance_data_check.py b/checkmates/data_checks/checks/no_variance_data_check.py index f85a509..0af2496 100644 --- a/checkmates/data_checks/checks/no_variance_data_check.py +++ b/checkmates/data_checks/checks/no_variance_data_check.py @@ -271,4 +271,4 @@ def validate(self, X, y=None): ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/null_data_check.py b/checkmates/data_checks/checks/null_data_check.py index 6ba106f..3250077 100644 --- a/checkmates/data_checks/checks/null_data_check.py +++ b/checkmates/data_checks/checks/null_data_check.py @@ -370,4 +370,4 @@ def get_null_row_information(X, pct_null_row_threshold=0.0): highly_null_rows = percent_null_rows[ percent_null_rows >= pct_null_row_threshold ] - return highly_null_rows \ No newline at end of file + return highly_null_rows diff --git a/checkmates/data_checks/checks/outliers_data_check.py b/checkmates/data_checks/checks/outliers_data_check.py index 45d7d2e..28b34fc 100644 --- a/checkmates/data_checks/checks/outliers_data_check.py +++ b/checkmates/data_checks/checks/outliers_data_check.py @@ -234,4 +234,4 @@ def _no_outlier_prob(num_records: int, pct_outliers: float) -> float: # calculate and return the probability of no true outliers for a gamma # cumulative density function prob_val = 1.0 - gamma.cdf(pct_outliers, shape_param, scale=scale_param) - return prob_val \ No newline at end of file + return prob_val diff --git a/checkmates/data_checks/checks/sparsity_data_check.py b/checkmates/data_checks/checks/sparsity_data_check.py index 95a5391..9409e8b 100644 --- a/checkmates/data_checks/checks/sparsity_data_check.py +++ b/checkmates/data_checks/checks/sparsity_data_check.py @@ -140,4 +140,4 @@ def sparsity_score(col, count_threshold=10): counts = col.value_counts() score = sum(counts > count_threshold) / counts.size - return score \ No newline at end of file + return score diff --git a/checkmates/data_checks/checks/target_distribution_data_check.py b/checkmates/data_checks/checks/target_distribution_data_check.py index 80051c6..4f38349 100644 --- a/checkmates/data_checks/checks/target_distribution_data_check.py +++ b/checkmates/data_checks/checks/target_distribution_data_check.py @@ -162,4 +162,4 @@ def _detect_log_distribution_helper(y): # with outliers dropped, then it would imply that the log transformed target has more of a normal distribution if norm_test_log.pvalue >= norm_test_og.pvalue: return True, normalization_test_string, norm_test_og - return False, normalization_test_string, norm_test_og \ No newline at end of file + return False, normalization_test_string, norm_test_og diff --git a/checkmates/data_checks/checks/target_leakage_data_check.py b/checkmates/data_checks/checks/target_leakage_data_check.py index dcd21b2..2c36769 100644 --- a/checkmates/data_checks/checks/target_leakage_data_check.py +++ b/checkmates/data_checks/checks/target_leakage_data_check.py @@ -169,4 +169,4 @@ def validate(self, X, y): ], ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/ts_parameters_data_check.py b/checkmates/data_checks/checks/ts_parameters_data_check.py index 5ab7b0e..5d76638 100644 --- a/checkmates/data_checks/checks/ts_parameters_data_check.py +++ b/checkmates/data_checks/checks/ts_parameters_data_check.py @@ -98,4 +98,4 @@ def validate(self, X, y=None): }, ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/ts_splitting_data_check.py b/checkmates/data_checks/checks/ts_splitting_data_check.py index 7664adf..9c66dce 100644 --- a/checkmates/data_checks/checks/ts_splitting_data_check.py +++ b/checkmates/data_checks/checks/ts_splitting_data_check.py @@ -103,4 +103,4 @@ def validate(self, X, y): }, ).to_dict(), ) - return messages \ No newline at end of file + return messages diff --git a/checkmates/data_checks/checks/uniqueness_data_check.py b/checkmates/data_checks/checks/uniqueness_data_check.py index 095e594..22784d1 100644 --- a/checkmates/data_checks/checks/uniqueness_data_check.py +++ b/checkmates/data_checks/checks/uniqueness_data_check.py @@ -182,4 +182,4 @@ def uniqueness_score(col, drop_na=True): ) square_counts = norm_counts**2 score = 1 - square_counts.sum() - return score \ No newline at end of file + return score diff --git a/checkmates/exceptions/__init__.py b/checkmates/exceptions/__init__.py index 5948d61..cafcc6c 100644 --- a/checkmates/exceptions/__init__.py +++ b/checkmates/exceptions/__init__.py @@ -4,5 +4,5 @@ MissingComponentError, ValidationErrorCode, ObjectiveCreationError, - ObjectiveNotFoundError + ObjectiveNotFoundError, ) diff --git a/checkmates/exceptions/exceptions.py b/checkmates/exceptions/exceptions.py index 0cf3fd2..1fa2540 100644 --- a/checkmates/exceptions/exceptions.py +++ b/checkmates/exceptions/exceptions.py @@ -7,14 +7,17 @@ class MissingComponentError(Exception): pass + class ObjectiveNotFoundError(Exception): """Exception to raise when specified objective does not exist.""" pass + class ObjectiveCreationError(Exception): """Exception when get_objective tries to instantiate an objective and required args are not provided.""" + class DataCheckInitError(Exception): """Exception raised when a data check can't initialize with the parameters given.""" diff --git a/checkmates/objectives/__init__.py b/checkmates/objectives/__init__.py index 92d200f..31e803e 100644 --- a/checkmates/objectives/__init__.py +++ b/checkmates/objectives/__init__.py @@ -1,3 +1,5 @@ +"""General Directory for CheckMates Objectives.""" + from checkmates.objectives.objective_base import ObjectiveBase from checkmates.objectives.regression_objective import RegressionObjective @@ -8,4 +10,4 @@ from checkmates.objectives.standard_metrics import RootMeanSquaredLogError -from checkmates.objectives.standard_metrics import MeanSquaredLogError \ No newline at end of file +from checkmates.objectives.standard_metrics import MeanSquaredLogError diff --git a/checkmates/objectives/objective_base.py b/checkmates/objectives/objective_base.py index a297eb8..f856457 100644 --- a/checkmates/objectives/objective_base.py +++ b/checkmates/objectives/objective_base.py @@ -214,4 +214,4 @@ def calculate_percent_difference(cls, score, baseline_score): @classmethod def is_defined_for_problem_type(cls, problem_type): """Returns whether or not an objective is defined for a problem type.""" - return handle_problem_types(problem_type) in cls.problem_types \ No newline at end of file + return handle_problem_types(problem_type) in cls.problem_types diff --git a/checkmates/objectives/regression_objective.py b/checkmates/objectives/regression_objective.py index ff7e78e..52aff70 100644 --- a/checkmates/objectives/regression_objective.py +++ b/checkmates/objectives/regression_objective.py @@ -7,4 +7,4 @@ class RegressionObjective(ObjectiveBase): """Base class for all regression objectives.""" problem_types = [ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION] - """[ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]""" \ No newline at end of file + """[ProblemTypes.REGRESSION, ProblemTypes.TIME_SERIES_REGRESSION]""" diff --git a/checkmates/objectives/standard_metrics.py b/checkmates/objectives/standard_metrics.py index fe72e3c..cc9eefa 100644 --- a/checkmates/objectives/standard_metrics.py +++ b/checkmates/objectives/standard_metrics.py @@ -6,6 +6,7 @@ from checkmates.objectives.regression_objective import RegressionObjective from checkmates.utils import classproperty + class RootMeanSquaredLogError(RegressionObjective): """Root mean squared log error for regression. @@ -97,4 +98,4 @@ def objective_function( @classproperty def positive_only(self): """If True, this objective is only valid for positive data.""" - return True \ No newline at end of file + return True diff --git a/checkmates/objectives/utils.py b/checkmates/objectives/utils.py index 0759169..c2fdb47 100644 --- a/checkmates/objectives/utils.py +++ b/checkmates/objectives/utils.py @@ -2,8 +2,9 @@ from checkmates import objectives from checkmates.exceptions import ObjectiveCreationError, ObjectiveNotFoundError from checkmates.objectives.objective_base import ObjectiveBase -from checkmates.utils.gen_utils import _get_subclasses from checkmates.problem_types import handle_problem_types +from checkmates.utils.gen_utils import _get_subclasses + def get_non_core_objectives(): """Get non-core objective classes. @@ -19,6 +20,7 @@ def get_non_core_objectives(): objectives.RootMeanSquaredLogError, ] + def _all_objectives_dict(): all_objectives = _get_subclasses(ObjectiveBase) objectives_dict = {} @@ -28,6 +30,7 @@ def _all_objectives_dict(): objectives_dict[objective.name.lower()] = objective return objectives_dict + def get_objective(objective, return_instance=False, **kwargs): """Returns the Objective class corresponding to a given objective name. @@ -76,6 +79,7 @@ def get_objective(objective, return_instance=False, **kwargs): return objective_class + def get_default_primary_search_objective(problem_type): """Get the default primary search objective for a problem type. @@ -96,6 +100,7 @@ def get_default_primary_search_objective(problem_type): }[problem_type.value] return get_objective(objective_name, return_instance=True) + def get_core_objectives(problem_type): """Returns all core objective instances associated with the given problem type. diff --git a/checkmates/utils/gen_utils.py b/checkmates/utils/gen_utils.py index e8e2b19..7a18387 100644 --- a/checkmates/utils/gen_utils.py +++ b/checkmates/utils/gen_utils.py @@ -4,6 +4,7 @@ logger = logging.getLogger(__name__) + def _get_subclasses(base_class): """Gets all of the leaf nodes in the hiearchy tree for a given base class. @@ -79,6 +80,7 @@ def __get__(self, _, klass): """Get property value.""" return self.func(klass) + def contains_all_ts_parameters(problem_configuration): """Validates that the problem configuration contains all required keys. @@ -108,6 +110,7 @@ def contains_all_ts_parameters(problem_configuration): ("is_valid", "msg", "smallest_split_size", "max_window_size", "n_obs", "n_splits"), ) + def are_ts_parameters_valid_for_split( gap, max_delay, diff --git a/tests/conftest.py b/tests/conftest.py index 6fccbb8..ecd0eef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,9 @@ import pandas as pd import pytest import woodwork as ww -from woodwork import logical_types as ww_logical_types from sklearn import datasets +from woodwork import logical_types as ww_logical_types + from checkmates.objectives.utils import get_core_objectives from checkmates.problem_types import ProblemTypes @@ -117,6 +118,7 @@ def X_y_binary(): y = ww.init_series(pd.Series(y), logical_type="integer") return X, y + @pytest.fixture def X_y_regression(): X, y = datasets.make_regression( @@ -130,10 +132,12 @@ def X_y_regression(): y = ww.init_series(pd.Series(y), logical_type="double") return X, y + @pytest.fixture def time_series_core_objectives(): return get_core_objectives(ProblemTypes.TIME_SERIES_REGRESSION) + @pytest.fixture def make_data_type(): """Helper function to convert numpy or pandas input to the appropriate type for tests.""" diff --git a/tests/data_checks_tests/test_class_imbalance_data_check.py b/tests/data_checks_tests/test_class_imbalance_data_check.py index a7dc8be..960e2d3 100644 --- a/tests/data_checks_tests/test_class_imbalance_data_check.py +++ b/tests/data_checks_tests/test_class_imbalance_data_check.py @@ -584,4 +584,4 @@ def test_class_imbalance_large_multiclass(test_size): ).to_dict(), ] - assert class_imbalance_check.validate(X, y_imbalanced_multiclass_nan) == [] \ No newline at end of file + assert class_imbalance_check.validate(X, y_imbalanced_multiclass_nan) == [] diff --git a/tests/data_checks_tests/test_datetime_format_data_check.py b/tests/data_checks_tests/test_datetime_format_data_check.py index 2354cd2..a432cb4 100644 --- a/tests/data_checks_tests/test_datetime_format_data_check.py +++ b/tests/data_checks_tests/test_datetime_format_data_check.py @@ -540,4 +540,4 @@ def test_datetime_many_duplicates_and_nans(): dc = DateTimeFormatDataCheck(datetime_column="date") result = dc.validate(X, y) - assert result[2]["code"] == "DATETIME_NO_FREQUENCY_INFERRED" \ No newline at end of file + assert result[2]["code"] == "DATETIME_NO_FREQUENCY_INFERRED" diff --git a/tests/data_checks_tests/test_invalid_target_data_check.py b/tests/data_checks_tests/test_invalid_target_data_check.py index 2cadeda..3c46623 100644 --- a/tests/data_checks_tests/test_invalid_target_data_check.py +++ b/tests/data_checks_tests/test_invalid_target_data_check.py @@ -3,7 +3,6 @@ import pytest import woodwork as ww -from checkmates.objectives.utils import get_default_primary_search_objective from checkmates.data_checks import ( DataCheckActionCode, DataCheckActionOption, @@ -19,7 +18,13 @@ MeanSquaredLogError, RootMeanSquaredLogError, ) -from checkmates.problem_types import ProblemTypes, is_binary, is_multiclass, is_regression +from checkmates.objectives.utils import get_default_primary_search_objective +from checkmates.problem_types import ( + ProblemTypes, + is_binary, + is_multiclass, + is_regression, +) from checkmates.utils.woodwork_utils import numeric_and_boolean_ww invalid_targets_data_check_name = InvalidTargetDataCheck.name @@ -817,4 +822,4 @@ def test_invalid_target_data_check_action_for_all_null(problem_type): ).to_dict(), ] messages = invalid_targets_check.validate(X, y_all_null) - assert messages == expected \ No newline at end of file + assert messages == expected diff --git a/tests/data_checks_tests/test_multicollinearity_data_check.py b/tests/data_checks_tests/test_multicollinearity_data_check.py index dfd5382..2569ae0 100644 --- a/tests/data_checks_tests/test_multicollinearity_data_check.py +++ b/tests/data_checks_tests/test_multicollinearity_data_check.py @@ -125,4 +125,4 @@ def test_multicollinearity_data_check_input_formats(): multi_check = MulticollinearityDataCheck(threshold=0.9) # test empty pd.DataFrame - assert multi_check.validate(pd.DataFrame()) == [] \ No newline at end of file + assert multi_check.validate(pd.DataFrame()) == [] diff --git a/tests/data_checks_tests/test_no_variance_data_check.py b/tests/data_checks_tests/test_no_variance_data_check.py index 824e0d6..513333c 100644 --- a/tests/data_checks_tests/test_no_variance_data_check.py +++ b/tests/data_checks_tests/test_no_variance_data_check.py @@ -200,4 +200,4 @@ def test_no_variance_data_check_warnings( expected_validation_result, ): check = NoVarianceDataCheck(count_nan_as_value) - assert check.validate(X, y) == expected_validation_result \ No newline at end of file + assert check.validate(X, y) == expected_validation_result diff --git a/tests/data_checks_tests/test_null_data_check.py b/tests/data_checks_tests/test_null_data_check.py index 2e12bed..c6046a3 100644 --- a/tests/data_checks_tests/test_null_data_check.py +++ b/tests/data_checks_tests/test_null_data_check.py @@ -672,4 +672,4 @@ def test_null_data_check_datetime_highly_null_dropped(): ), ], ).to_dict(), - ] \ No newline at end of file + ] diff --git a/tests/data_checks_tests/test_outliers_data_check.py b/tests/data_checks_tests/test_outliers_data_check.py index 5dabf10..ca11415 100644 --- a/tests/data_checks_tests/test_outliers_data_check.py +++ b/tests/data_checks_tests/test_outliers_data_check.py @@ -253,4 +253,4 @@ def test_boxplot_stats(data_type): "low_indices": test[test < field_bounds[0]].index.tolist(), "high_indices": test[test > field_bounds[1]].index.tolist(), }, - } \ No newline at end of file + } diff --git a/tests/data_checks_tests/test_sparsity_data_check.py b/tests/data_checks_tests/test_sparsity_data_check.py index 3659ec0..79c0125 100644 --- a/tests/data_checks_tests/test_sparsity_data_check.py +++ b/tests/data_checks_tests/test_sparsity_data_check.py @@ -150,4 +150,4 @@ def test_sparsity_data_check_warnings(): ), ], ).to_dict(), - ] \ No newline at end of file + ] diff --git a/tests/data_checks_tests/test_target_distribution_data_check.py b/tests/data_checks_tests/test_target_distribution_data_check.py index ca3dfbf..e0f430e 100644 --- a/tests/data_checks_tests/test_target_distribution_data_check.py +++ b/tests/data_checks_tests/test_target_distribution_data_check.py @@ -130,4 +130,4 @@ # ), # ], # ).to_dict(), -# ] \ No newline at end of file +# ] diff --git a/tests/data_checks_tests/test_target_leakage_data_check.py b/tests/data_checks_tests/test_target_leakage_data_check.py index c712482..472065f 100644 --- a/tests/data_checks_tests/test_target_leakage_data_check.py +++ b/tests/data_checks_tests/test_target_leakage_data_check.py @@ -559,4 +559,4 @@ def test_target_leakage_target_all_matches_max(): X["e"] = [0, 1, 2, 3] * 10 leakage_check_all = TargetLeakageDataCheck(pct_corr_threshold=0.8, method="all") leakage_check_max = TargetLeakageDataCheck(pct_corr_threshold=0.8, method="max") - assert leakage_check_all.validate(X, y) == leakage_check_max.validate(X, y) \ No newline at end of file + assert leakage_check_all.validate(X, y) == leakage_check_max.validate(X, y) diff --git a/tests/data_checks_tests/test_ts_parameters_data_check.py b/tests/data_checks_tests/test_ts_parameters_data_check.py index 6f72b8e..f109e14 100644 --- a/tests/data_checks_tests/test_ts_parameters_data_check.py +++ b/tests/data_checks_tests/test_ts_parameters_data_check.py @@ -77,4 +77,4 @@ def test_time_series_param_data_check( } assert messages[0]["code"] == code else: - assert messages == [] \ No newline at end of file + assert messages == [] diff --git a/tests/data_checks_tests/test_ts_splitting_data_check.py b/tests/data_checks_tests/test_ts_splitting_data_check.py index 0460fbd..c881dc8 100644 --- a/tests/data_checks_tests/test_ts_splitting_data_check.py +++ b/tests/data_checks_tests/test_ts_splitting_data_check.py @@ -54,4 +54,4 @@ def test_time_series_param_data_check(is_valid, problem_type): } assert messages[0]["code"] == code else: - assert messages == [] \ No newline at end of file + assert messages == [] diff --git a/tests/data_checks_tests/test_uniqueness_data_check.py b/tests/data_checks_tests/test_uniqueness_data_check.py index 671d0c4..f101f39 100644 --- a/tests/data_checks_tests/test_uniqueness_data_check.py +++ b/tests/data_checks_tests/test_uniqueness_data_check.py @@ -137,4 +137,4 @@ def test_uniqueness_data_check_warnings(): ), ], ).to_dict(), - ] \ No newline at end of file + ] From a0ad484d95c9085745fbcf102849da8892af0464 Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Tue, 15 Aug 2023 13:12:16 -0400 Subject: [PATCH 4/8] uncommented target distr test --- .../test_target_distribution_data_check.py | 266 +++++++++--------- 1 file changed, 133 insertions(+), 133 deletions(-) diff --git a/tests/data_checks_tests/test_target_distribution_data_check.py b/tests/data_checks_tests/test_target_distribution_data_check.py index e0f430e..5628b9e 100644 --- a/tests/data_checks_tests/test_target_distribution_data_check.py +++ b/tests/data_checks_tests/test_target_distribution_data_check.py @@ -1,133 +1,133 @@ -# import numpy as np -# import pandas as pd -# import pytest -# from scipy.stats import jarque_bera, lognorm, norm, shapiro - -# from checkmates.data_checks import ( -# DataCheckActionCode, -# DataCheckActionOption, -# DataCheckError, -# DataCheckMessageCode, -# DataCheckWarning, -# TargetDistributionDataCheck, -# ) -# from checkmates.utils import infer_feature_types - -# target_dist_check_name = TargetDistributionDataCheck.name - - -# def test_target_distribution_data_check_no_y(X_y_regression): -# X, y = X_y_regression -# y = None - -# target_dist_check = TargetDistributionDataCheck() - -# assert target_dist_check.validate(X, y) == [ -# DataCheckError( -# message="Target is None", -# data_check_name=target_dist_check_name, -# message_code=DataCheckMessageCode.TARGET_IS_NONE, -# details={}, -# ).to_dict(), -# ] - - -# @pytest.mark.parametrize("target_type", ["boolean", "categorical", "integer", "double"]) -# def test_target_distribution_data_check_unsupported_target_type(target_type): -# X = pd.DataFrame(range(5)) - -# if target_type == "boolean": -# y = pd.Series([True, False] * 5) -# elif target_type == "categorical": -# y = pd.Series(["One", "Two", "Three", "Four", "Five"] * 2) -# elif target_type == "integer": -# y = [-1, -3, -5, 4, -2, 4, -4, 2, 1, 1] -# else: -# y = [9.2, 7.66, 4.93, 3.29, 4.06, -1.28, 4.95, 6.77, 9.07, 7.67] - -# y = infer_feature_types(y) - -# target_dist_check = TargetDistributionDataCheck() - -# if target_type in ["integer", "double"]: -# assert target_dist_check.validate(X, y) == [] -# else: -# assert target_dist_check.validate(X, y) == [ -# DataCheckError( -# message=f"Target is unsupported {y.ww.logical_type.type_string} type. Valid Woodwork logical types include: integer, double, age, age_fractional", -# data_check_name=target_dist_check_name, -# message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE, -# details={"unsupported_type": y.ww.logical_type.type_string}, -# ).to_dict(), -# ] - - -# @pytest.mark.parametrize("data_type", ["positive", "mixed", "negative"]) -# @pytest.mark.parametrize("distribution", ["normal", "lognormal", "very_lognormal"]) -# @pytest.mark.parametrize( -# "size,name,statistic", -# [(10000, "jarque_bera", jarque_bera), (5000, "shapiro", shapiro)], -# ) -# def test_target_distribution_data_check_warning_action( -# size, -# name, -# statistic, -# distribution, -# data_type, -# X_y_regression, -# ): -# X, y = X_y_regression -# # set this to avoid flaky tests. This is primarily because when we have smaller samples, -# # once we remove values outside 3 st.devs, the distribution can begin to look more normal -# random_state = 2 -# target_dist_check = TargetDistributionDataCheck() - -# if distribution == "normal": -# y = norm.rvs(loc=3, size=size, random_state=random_state) -# elif distribution == "lognormal": -# y = lognorm.rvs(0.4, size=size, random_state=random_state) -# else: -# # Will have a p-value of 0 thereby rejecting the null hypothesis even after log transforming -# # This is essentially just checking the = of the statistic's "log.pvalue >= og.pvalue" -# y = lognorm.rvs(s=1, loc=1, scale=1, size=size, random_state=random_state) - -# y = np.round(y, 6) - -# if data_type == "negative": -# y = -np.abs(y) -# elif data_type == "mixed": -# y = y - 1.2 - -# if distribution == "normal": -# assert target_dist_check.validate(X, y) == [] -# else: -# target_dist_ = target_dist_check.validate(X, y) - -# if any(y <= 0): -# y = y + abs(y.min()) + 1 -# y = y[y < (y.mean() + 3 * round(y.std(), 3))] -# test_og = statistic(y) - -# details = { -# "normalization_method": name, -# "statistic": round(test_og.statistic, 1), -# "p-value": round(test_og.pvalue, 3), -# } -# assert target_dist_ == [ -# DataCheckWarning( -# message="Target may have a lognormal distribution.", -# data_check_name=target_dist_check_name, -# message_code=DataCheckMessageCode.TARGET_LOGNORMAL_DISTRIBUTION, -# details=details, -# action_options=[ -# DataCheckActionOption( -# DataCheckActionCode.TRANSFORM_TARGET, -# data_check_name=target_dist_check_name, -# metadata={ -# "is_target": True, -# "transformation_strategy": "lognormal", -# }, -# ), -# ], -# ).to_dict(), -# ] +import numpy as np +import pandas as pd +import pytest +from scipy.stats import jarque_bera, lognorm, norm, shapiro + +from checkmates.data_checks import ( + DataCheckActionCode, + DataCheckActionOption, + DataCheckError, + DataCheckMessageCode, + DataCheckWarning, + TargetDistributionDataCheck, +) +from checkmates.utils import infer_feature_types + +target_dist_check_name = TargetDistributionDataCheck.name + + +def test_target_distribution_data_check_no_y(X_y_regression): + X, y = X_y_regression + y = None + + target_dist_check = TargetDistributionDataCheck() + + assert target_dist_check.validate(X, y) == [ + DataCheckError( + message="Target is None", + data_check_name=target_dist_check_name, + message_code=DataCheckMessageCode.TARGET_IS_NONE, + details={}, + ).to_dict(), + ] + + +@pytest.mark.parametrize("target_type", ["boolean", "categorical", "integer", "double"]) +def test_target_distribution_data_check_unsupported_target_type(target_type): + X = pd.DataFrame(range(5)) + + if target_type == "boolean": + y = pd.Series([True, False] * 5) + elif target_type == "categorical": + y = pd.Series(["One", "Two", "Three", "Four", "Five"] * 2) + elif target_type == "integer": + y = [-1, -3, -5, 4, -2, 4, -4, 2, 1, 1] + else: + y = [9.2, 7.66, 4.93, 3.29, 4.06, -1.28, 4.95, 6.77, 9.07, 7.67] + + y = infer_feature_types(y) + + target_dist_check = TargetDistributionDataCheck() + + if target_type in ["integer", "double"]: + assert target_dist_check.validate(X, y) == [] + else: + assert target_dist_check.validate(X, y) == [ + DataCheckError( + message=f"Target is unsupported {y.ww.logical_type.type_string} type. Valid Woodwork logical types include: integer, double, age, age_fractional", + data_check_name=target_dist_check_name, + message_code=DataCheckMessageCode.TARGET_UNSUPPORTED_TYPE, + details={"unsupported_type": y.ww.logical_type.type_string}, + ).to_dict(), + ] + + +@pytest.mark.parametrize("data_type", ["positive", "mixed", "negative"]) +@pytest.mark.parametrize("distribution", ["normal", "lognormal", "very_lognormal"]) +@pytest.mark.parametrize( + "size,name,statistic", + [(10000, "jarque_bera", jarque_bera), (5000, "shapiro", shapiro)], +) +def test_target_distribution_data_check_warning_action( + size, + name, + statistic, + distribution, + data_type, + X_y_regression, +): + X, y = X_y_regression + # set this to avoid flaky tests. This is primarily because when we have smaller samples, + # once we remove values outside 3 st.devs, the distribution can begin to look more normal + random_state = 2 + target_dist_check = TargetDistributionDataCheck() + + if distribution == "normal": + y = norm.rvs(loc=3, size=size, random_state=random_state) + elif distribution == "lognormal": + y = lognorm.rvs(0.4, size=size, random_state=random_state) + else: + # Will have a p-value of 0 thereby rejecting the null hypothesis even after log transforming + # This is essentially just checking the = of the statistic's "log.pvalue >= og.pvalue" + y = lognorm.rvs(s=1, loc=1, scale=1, size=size, random_state=random_state) + + y = np.round(y, 6) + + if data_type == "negative": + y = -np.abs(y) + elif data_type == "mixed": + y = y - 1.2 + + if distribution == "normal": + assert target_dist_check.validate(X, y) == [] + else: + target_dist_ = target_dist_check.validate(X, y) + + if any(y <= 0): + y = y + abs(y.min()) + 1 + y = y[y < (y.mean() + 3 * round(y.std(), 3))] + test_og = statistic(y) + + details = { + "normalization_method": name, + "statistic": round(test_og.statistic, 1), + "p-value": round(test_og.pvalue, 3), + } + assert target_dist_ == [ + DataCheckWarning( + message="Target may have a lognormal distribution.", + data_check_name=target_dist_check_name, + message_code=DataCheckMessageCode.TARGET_LOGNORMAL_DISTRIBUTION, + details=details, + action_options=[ + DataCheckActionOption( + DataCheckActionCode.TRANSFORM_TARGET, + data_check_name=target_dist_check_name, + metadata={ + "is_target": True, + "transformation_strategy": "lognormal", + }, + ), + ], + ).to_dict(), + ] From d05d032612e44fd7cce48d3c68071e188f49f42a Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Wed, 16 Aug 2023 10:58:02 -0400 Subject: [PATCH 5/8] invalid_target data_check added --- checkmates/objectives/__init__.py | 3 + .../binary_classification_objective.py | 84 ++++++++++++ .../multiclass_classification_objective.py | 10 ++ checkmates/objectives/standard_metrics.py | 121 ++++++++++++++++++ checkmates/objectives/utils.py | 14 +- 5 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 checkmates/objectives/binary_classification_objective.py create mode 100644 checkmates/objectives/multiclass_classification_objective.py diff --git a/checkmates/objectives/__init__.py b/checkmates/objectives/__init__.py index 31e803e..cfda5d1 100644 --- a/checkmates/objectives/__init__.py +++ b/checkmates/objectives/__init__.py @@ -11,3 +11,6 @@ from checkmates.objectives.standard_metrics import RootMeanSquaredLogError from checkmates.objectives.standard_metrics import MeanSquaredLogError + +from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective +from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective \ No newline at end of file diff --git a/checkmates/objectives/binary_classification_objective.py b/checkmates/objectives/binary_classification_objective.py new file mode 100644 index 0000000..5bfbdd0 --- /dev/null +++ b/checkmates/objectives/binary_classification_objective.py @@ -0,0 +1,84 @@ +"""Base class for all binary classification objectives.""" +import numpy as np +from scipy.optimize import differential_evolution + +from checkmates.objectives.objective_base import ObjectiveBase +from checkmates.problem_types import ProblemTypes + + +class BinaryClassificationObjective(ObjectiveBase): + """Base class for all binary classification objectives.""" + + problem_types = [ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY] + + """[ProblemTypes.BINARY, ProblemTypes.TIME_SERIES_BINARY]""" + + @property + def can_optimize_threshold(cls): + """Returns a boolean determining if we can optimize the binary classification objective threshold. + + This will be false for any objective that works directly with + predicted probabilities, like log loss and AUC. Otherwise, it + will be true. + + Returns: + bool: Whether or not an objective can be optimized. + """ + return not cls.score_needs_proba + + def optimize_threshold(self, ypred_proba, y_true, X=None): + """Learn a binary classification threshold which optimizes the current objective. + + Args: + ypred_proba (pd.Series): The classifier's predicted probabilities + y_true (pd.Series): The ground truth for the predictions. + X (pd.DataFrame, optional): Any extra columns that are needed from training data. + + Returns: + Optimal threshold for this objective. + + Raises: + RuntimeError: If objective cannot be optimized. + """ + ypred_proba = self._standardize_input_type(ypred_proba) + y_true = self._standardize_input_type(y_true) + if X is not None: + X = self._standardize_input_type(X) + + if not self.can_optimize_threshold: + raise RuntimeError("Trying to optimize objective that can't be optimized!") + + def cost(threshold): + y_predicted = self.decision_function( + ypred_proba=ypred_proba, + threshold=threshold[0], + X=X, + ) + cost = self.objective_function(y_true, y_predicted, X=X) + return -cost if self.greater_is_better else cost + + optimal = differential_evolution(cost, bounds=[(0, 1)], seed=0, maxiter=250) + + return optimal.x[0] + + def decision_function(self, ypred_proba, threshold=0.5, X=None): + """Apply a learned threshold to predicted probabilities to get predicted classes. + + Args: + ypred_proba (pd.Series, np.ndarray): The classifier's predicted probabilities + threshold (float, optional): Threshold used to make a prediction. Defaults to 0.5. + X (pd.DataFrame, optional): Any extra columns that are needed from training data. + + Returns: + predictions + """ + ypred_proba = self._standardize_input_type(ypred_proba) + return ypred_proba > threshold + + def validate_inputs(self, y_true, y_predicted): + """Validate inputs for scoring.""" + super().validate_inputs(y_true, y_predicted) + if len(np.unique(y_true)) > 2: + raise ValueError("y_true contains more than two unique values") + if len(np.unique(y_predicted)) > 2 and not self.score_needs_proba: + raise ValueError("y_predicted contains more than two unique values") \ No newline at end of file diff --git a/checkmates/objectives/multiclass_classification_objective.py b/checkmates/objectives/multiclass_classification_objective.py new file mode 100644 index 0000000..abf9c73 --- /dev/null +++ b/checkmates/objectives/multiclass_classification_objective.py @@ -0,0 +1,10 @@ +"""Base class for all multiclass classification objectives.""" +from checkmates.objectives.objective_base import ObjectiveBase +from checkmates.problem_types import ProblemTypes + + +class MulticlassClassificationObjective(ObjectiveBase): + """Base class for all multiclass classification objectives.""" + + problem_types = [ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS] + """[ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS]""" \ No newline at end of file diff --git a/checkmates/objectives/standard_metrics.py b/checkmates/objectives/standard_metrics.py index cc9eefa..641d331 100644 --- a/checkmates/objectives/standard_metrics.py +++ b/checkmates/objectives/standard_metrics.py @@ -5,6 +5,127 @@ from checkmates.objectives.regression_objective import RegressionObjective from checkmates.utils import classproperty +from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective +from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective + + +class LogLossBinary(BinaryClassificationObjective): + """Log Loss for binary classification. + + Example: + >>> y_true = pd.Series([0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1]) + >>> y_pred = pd.Series([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]) + >>> np.testing.assert_almost_equal(LogLossBinary().objective_function(y_true, y_pred), 19.6601745) + """ + + name = "Log Loss Binary" + greater_is_better = False + score_needs_proba = True + perfect_score = 0.0 + is_bounded_like_percentage = False # Range [0, Inf) + expected_range = [0, 1] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for log loss for binary classification.""" + return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight) + +class LogLossMulticlass(MulticlassClassificationObjective): + """Log Loss for multiclass classification. + + Example: + >>> y_true = [0, 1, 2, 0, 2, 1] + >>> y_pred = [[0.7, 0.2, 0.1], + ... [0.3, 0.5, 0.2], + ... [0.1, 0.3, 0.6], + ... [0.9, 0.1, 0.0], + ... [0.3, 0.1, 0.6], + ... [0.5, 0.5, 0.0]] + >>> np.testing.assert_almost_equal(LogLossMulticlass().objective_function(y_true, y_pred), 0.4783301) + """ + + name = "Log Loss Multiclass" + greater_is_better = False + score_needs_proba = True + perfect_score = 0.0 + is_bounded_like_percentage = False # Range [0, Inf) + expected_range = [0, 1] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for log loss for multiclass classification.""" + return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight) + +class R2(RegressionObjective): + """Coefficient of determination for regression. + + Example: + >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2]) + >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75]) + >>> np.testing.assert_almost_equal(R2().objective_function(y_true, y_pred), 0.7638036) + """ + + name = "R2" + greater_is_better = True + score_needs_proba = False + perfect_score = 1 + is_bounded_like_percentage = False # Range (-Inf, 1] + expected_range = [-1, 1] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for coefficient of determination for regression.""" + return metrics.r2_score(y_true, y_predicted, sample_weight=sample_weight) + +class MedianAE(RegressionObjective): + """Median absolute error for regression. + + Example: + >>> y_true = pd.Series([1.5, 2, 3, 1, 0.5, 1, 2.5, 2.5, 1, 0.5, 2]) + >>> y_pred = pd.Series([1.5, 2.5, 2, 1, 0.5, 1, 3, 2.25, 0.75, 0.25, 1.75]) + >>> np.testing.assert_almost_equal(MedianAE().objective_function(y_true, y_pred), 0.25) + """ + + name = "MedianAE" + greater_is_better = False + score_needs_proba = False + perfect_score = 0.0 + is_bounded_like_percentage = False # Range [0, Inf) + expected_range = [0, float("inf")] + + def objective_function( + self, + y_true, + y_predicted, + y_train=None, + X=None, + sample_weight=None, + ): + """Objective function for median absolute error for regression.""" + return metrics.median_absolute_error( + y_true, + y_predicted, + sample_weight=sample_weight, + ) + class RootMeanSquaredLogError(RegressionObjective): diff --git a/checkmates/objectives/utils.py b/checkmates/objectives/utils.py index c2fdb47..4c42548 100644 --- a/checkmates/objectives/utils.py +++ b/checkmates/objectives/utils.py @@ -1,4 +1,4 @@ -"""Utility methods for EvalML objectives.""" +"""Utility methods for CheckMates objectives.""" from checkmates import objectives from checkmates.exceptions import ObjectiveCreationError, ObjectiveNotFoundError from checkmates.objectives.objective_base import ObjectiveBase @@ -20,12 +20,20 @@ def get_non_core_objectives(): objectives.RootMeanSquaredLogError, ] +def get_all_objective_names(): + """Get a list of the names of all objectives. + + Returns: + list (str): Objective names + """ + all_objectives_dict = _all_objectives_dict() + return list(all_objectives_dict.keys()) def _all_objectives_dict(): all_objectives = _get_subclasses(ObjectiveBase) objectives_dict = {} for objective in all_objectives: - if "evalml.objectives" not in objective.__module__: + if "checkmates.objectives" not in objective.__module__: continue objectives_dict[objective.name.lower()] = objective return objectives_dict @@ -63,7 +71,7 @@ def get_objective(objective, return_instance=False, **kwargs): if objective.lower() not in all_objectives_dict: raise ObjectiveNotFoundError( f"{objective} is not a valid Objective! " - "Use evalml.objectives.get_all_objective_names() " + "Use checkmates.objectives.get_all_objective_names() " "to get a list of all valid objective names. ", ) From 49aa7c347474fe6bcfe2642c62d04bce28c8a8e2 Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Wed, 16 Aug 2023 11:06:34 -0400 Subject: [PATCH 6/8] release notes updated --- docs/source/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 876e510..c8f1dd6 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -3,6 +3,7 @@ Release Notes **Future Releases** * Enhancements * Added all datachecks except `invalid_target_data_check`` along with tests and utils, migrated over from `EvalML` :pr:`15` + * Added `invalid_target_data_check` along with all tests, utils, and objectives, migrated from `EvalML` :pr:`17` * Fixes * Changes * Documentation Changes From b6aab8ef9a9893771ba7fe050110fa8557b19e84 Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Wed, 16 Aug 2023 11:07:06 -0400 Subject: [PATCH 7/8] lint fix --- checkmates/objectives/__init__.py | 8 ++++++-- .../objectives/binary_classification_objective.py | 2 +- .../multiclass_classification_objective.py | 2 +- checkmates/objectives/standard_metrics.py | 12 +++++++++--- checkmates/objectives/utils.py | 2 ++ 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/checkmates/objectives/__init__.py b/checkmates/objectives/__init__.py index cfda5d1..ba6a55b 100644 --- a/checkmates/objectives/__init__.py +++ b/checkmates/objectives/__init__.py @@ -12,5 +12,9 @@ from checkmates.objectives.standard_metrics import RootMeanSquaredLogError from checkmates.objectives.standard_metrics import MeanSquaredLogError -from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective -from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective \ No newline at end of file +from checkmates.objectives.binary_classification_objective import ( + BinaryClassificationObjective, +) +from checkmates.objectives.multiclass_classification_objective import ( + MulticlassClassificationObjective, +) diff --git a/checkmates/objectives/binary_classification_objective.py b/checkmates/objectives/binary_classification_objective.py index 5bfbdd0..2a868eb 100644 --- a/checkmates/objectives/binary_classification_objective.py +++ b/checkmates/objectives/binary_classification_objective.py @@ -81,4 +81,4 @@ def validate_inputs(self, y_true, y_predicted): if len(np.unique(y_true)) > 2: raise ValueError("y_true contains more than two unique values") if len(np.unique(y_predicted)) > 2 and not self.score_needs_proba: - raise ValueError("y_predicted contains more than two unique values") \ No newline at end of file + raise ValueError("y_predicted contains more than two unique values") diff --git a/checkmates/objectives/multiclass_classification_objective.py b/checkmates/objectives/multiclass_classification_objective.py index abf9c73..da9fa2e 100644 --- a/checkmates/objectives/multiclass_classification_objective.py +++ b/checkmates/objectives/multiclass_classification_objective.py @@ -7,4 +7,4 @@ class MulticlassClassificationObjective(ObjectiveBase): """Base class for all multiclass classification objectives.""" problem_types = [ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS] - """[ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS]""" \ No newline at end of file + """[ProblemTypes.MULTICLASS, ProblemTypes.TIME_SERIES_MULTICLASS]""" diff --git a/checkmates/objectives/standard_metrics.py b/checkmates/objectives/standard_metrics.py index 641d331..b263133 100644 --- a/checkmates/objectives/standard_metrics.py +++ b/checkmates/objectives/standard_metrics.py @@ -3,10 +3,14 @@ import pandas as pd from sklearn import metrics +from checkmates.objectives.binary_classification_objective import ( + BinaryClassificationObjective, +) +from checkmates.objectives.multiclass_classification_objective import ( + MulticlassClassificationObjective, +) from checkmates.objectives.regression_objective import RegressionObjective from checkmates.utils import classproperty -from checkmates.objectives.binary_classification_objective import BinaryClassificationObjective -from checkmates.objectives.multiclass_classification_objective import MulticlassClassificationObjective class LogLossBinary(BinaryClassificationObjective): @@ -36,6 +40,7 @@ def objective_function( """Objective function for log loss for binary classification.""" return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight) + class LogLossMulticlass(MulticlassClassificationObjective): """Log Loss for multiclass classification. @@ -68,6 +73,7 @@ def objective_function( """Objective function for log loss for multiclass classification.""" return metrics.log_loss(y_true, y_predicted, sample_weight=sample_weight) + class R2(RegressionObjective): """Coefficient of determination for regression. @@ -95,6 +101,7 @@ def objective_function( """Objective function for coefficient of determination for regression.""" return metrics.r2_score(y_true, y_predicted, sample_weight=sample_weight) + class MedianAE(RegressionObjective): """Median absolute error for regression. @@ -127,7 +134,6 @@ def objective_function( ) - class RootMeanSquaredLogError(RegressionObjective): """Root mean squared log error for regression. diff --git a/checkmates/objectives/utils.py b/checkmates/objectives/utils.py index 4c42548..1ba882b 100644 --- a/checkmates/objectives/utils.py +++ b/checkmates/objectives/utils.py @@ -20,6 +20,7 @@ def get_non_core_objectives(): objectives.RootMeanSquaredLogError, ] + def get_all_objective_names(): """Get a list of the names of all objectives. @@ -29,6 +30,7 @@ def get_all_objective_names(): all_objectives_dict = _all_objectives_dict() return list(all_objectives_dict.keys()) + def _all_objectives_dict(): all_objectives = _get_subclasses(ObjectiveBase) objectives_dict = {} From ff9ff131518f1cb9572fb14d3c3f44b7e84a72de Mon Sep 17 00:00:00 2001 From: Nabil Fayak Date: Thu, 17 Aug 2023 11:11:47 -0400 Subject: [PATCH 8/8] release notes updated --- docs/source/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index c8f1dd6..c6a851b 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -3,7 +3,7 @@ Release Notes **Future Releases** * Enhancements * Added all datachecks except `invalid_target_data_check`` along with tests and utils, migrated over from `EvalML` :pr:`15` - * Added `invalid_target_data_check` along with all tests, utils, and objectives, migrated from `EvalML` :pr:`17` + * Added ``invalid_target_data_check`` along with all tests, utils, and objectives, migrated from ``EvalML`` :pr:`17` * Fixes * Changes * Documentation Changes