Intake lists

cytominer_eval/evaluate.py

@@ -17,6 +17,7 @@
     enrichment,
 )
 
+
 def evaluate(
     profiles: pd.DataFrame,
     features: List[str],
@@ -26,11 +27,11 @@ def evaluate(
     similarity_metric: str = "pearson",
     replicate_reproducibility_quantile: np.float = 0.95,
     replicate_reproducibility_return_median_cor: bool = False,
-    precision_recall_k: int = 10,
+    precision_recall_k: List[int] = [5, 10],
     grit_control_perts: List[str] = ["None"],
     grit_replicate_summary_method: str = "mean",
     mp_value_params: dict = {},
-    enrichment_percentile: float = 0.5,
+    enrichment_percentile: List[float] = [0.99, 0.98],
 ):
     r"""Evaluate profile quality and strength.
 
@@ -129,7 +130,7 @@ def evaluate(
         metric_result = precision_recall(
             similarity_melted_df=similarity_melted_df,
             replicate_groups=replicate_groups,
-            k=precision_recall_k,
+            k_list=precision_recall_k,
         )
     elif operation == "grit":
         metric_result = grit(

cytominer_eval/operations/enrichment.py

@@ -14,8 +14,10 @@
 
 
 def enrichment(
-    similarity_melted_df: pd.DataFrame, replicate_groups: List[str], percentile: 0.9,
-) -> dict:
+    similarity_melted_df: pd.DataFrame,
+    replicate_groups: List[str],
+    percentile: List[float],
+) -> pd.DataFrame:
     """Calculate the enrichment score. This score is based on the fisher exact odds score. Similar to the other functions, the closest connections are determined and checked with the replicates.
     This score effectively calculates how much better the distribution of correct connections is compared to random.
 
@@ -28,48 +30,54 @@ def enrichment(
     replicate_groups : List
         a list of metadata column names in the original profile dataframe to use as
         replicate columns.
-    percentile :  float
+    enrichment_percentile :  List of floats
         Determines what percentage of top connections used for the enrichment calculation.
 
     Returns
     -------
     dict
         percentile, threshold, odds ratio and p value
     """
-    # threshold based on percentile of top connections
-    threshold = similarity_melted_df.similarity_metric.quantile(percentile)
-
+    result = []
     replicate_truth_df = assign_replicates(
         similarity_melted_df=similarity_melted_df, replicate_groups=replicate_groups
     )
-    # calculate the individual components of the contingency tables
-    v11 = len(
-        replicate_truth_df.query(
-            "group_replicate==True and similarity_metric>@threshold"
+    # loop over all percentiles
+    for p in percentile:
+        # threshold based on percentile of top connections
+        threshold = similarity_melted_df.similarity_metric.quantile(p)
+
+        # calculate the individual components of the contingency tables
+        v11 = len(
+            replicate_truth_df.query(
+                "group_replicate==True and similarity_metric>@threshold"
+            )
         )
-    )
-    v12 = len(
-        replicate_truth_df.query(
-            "group_replicate==False and similarity_metric>@threshold"
+        v12 = len(
+            replicate_truth_df.query(
+                "group_replicate==False and similarity_metric>@threshold"
+            )
         )
-    )
-    v21 = len(
-        replicate_truth_df.query(
-            "group_replicate==True and similarity_metric<=@threshold"
+        v21 = len(
+            replicate_truth_df.query(
+                "group_replicate==True and similarity_metric<=@threshold"
+            )
         )
-    )
-    v22 = len(
-        replicate_truth_df.query(
-            "group_replicate==False and similarity_metric<=@threshold"
+        v22 = len(
+            replicate_truth_df.query(
+                "group_replicate==False and similarity_metric<=@threshold"
+            )
         )
-    )
 
-    v = np.asarray([[v11, v12], [v21, v22]])
-    r = scipy.stats.fisher_exact(v, alternative="greater")
-    result = {
-        "percentile": percentile,
-        "threshold": threshold,
-        "ods_ratio": r[0],
-        "p-value": r[1],
-    }
-    return result
+        v = np.asarray([[v11, v12], [v21, v22]])
+        r = scipy.stats.fisher_exact(v, alternative="greater")
+        result.append(
+            {
+                "enrichment_percentile": p,
+                "threshold": threshold,
+                "ods_ratio": r[0],
+                "p-value": r[1],
+            }
+        )
+    result_df = pd.DataFrame(result)
+    return result_df

cytominer_eval/operations/precision_recall.py

@@ -11,9 +11,7 @@
 
 
 def precision_recall(
-    similarity_melted_df: pd.DataFrame,
-    replicate_groups: List[str],
-    k: int,
+    similarity_melted_df: pd.DataFrame, replicate_groups: List[str], k_list: List[int],
 ) -> pd.DataFrame:
     """Determine the precision and recall at k for all unique replicate groups
     based on a predefined similarity metric (see cytominer_eval.transform.metric_melt)
@@ -27,7 +25,7 @@ def precision_recall(
     replicate_groups : List
         a list of metadata column names in the original profile dataframe to use as
         replicate columns.
-    k : int
+    k_list : List of ints
         an integer indicating how many pairwise comparisons to threshold.
 
     Returns
@@ -49,11 +47,14 @@ def precision_recall(
         "{x}{suf}".format(x=x, suf=pair_ids[list(pair_ids)[0]]["suffix"])
         for x in replicate_groups
     ]
-
-    # Calculate precision and recall for all groups
-    precision_recall_df = similarity_melted_df.groupby(replicate_group_cols).apply(
-        lambda x: calculate_precision_recall(x, k=k)
-    )
+    # iterate over all k
+    precision_recall_df = pd.DataFrame()
+    for k in k_list:
+        # Calculate precision and recall for all groups
+        precision_recall_df_at_k = similarity_melted_df.groupby(
+            replicate_group_cols
+        ).apply(lambda x: calculate_precision_recall(x, k=k))
+        precision_recall_df = precision_recall_df.append(precision_recall_df_at_k)
 
     # Rename the columns back to the replicate groups provided
     rename_cols = dict(zip(replicate_group_cols, replicate_groups))

cytominer_eval/tests/test_evaluate.py

@@ -141,7 +141,7 @@ def test_evaluate_precision_recall():
             replicate_groups=gene_groups,
             operation="precision_recall",
             similarity_metric="pearson",
-            precision_recall_k=k,
+            precision_recall_k=[k],
         )
 
         assert (
@@ -160,7 +160,7 @@ def test_evaluate_precision_recall():
             replicate_groups=["Metadata_broad_sample"],
             operation="precision_recall",
             similarity_metric="pearson",
-            precision_recall_k=k,
+            precision_recall_k=[k],
         )
 
         assert (

cytominer_eval/tests/test_operations/test_enrichment.py

@@ -42,37 +42,34 @@
 
 
 def test_enrichment():
-    result = []
-    for p in np.arange(1, 0.97, -0.005):
-        r = enrichment(
-            similarity_melted_df=similarity_melted_df,
-            replicate_groups=replicate_groups,
-            percentile=p,
-        )
-        result.append(r)
-    result_df = pd.DataFrame(result)
+    percent_list = np.arange(1, 0.97, -0.005)
+    result = enrichment(
+        similarity_melted_df=similarity_melted_df,
+        replicate_groups=replicate_groups,
+        percentile=percent_list,
+    )
 
     # check for correct shape and starts with 1.0
-    assert result_df.shape == (7, 4)
-    assert result_df.percentile[0] == 1.0
+    assert result.shape == (7, 4)
+    assert result.enrichment_percentile[0] == 1.0
     # check if the higher percentiles are larger than the small one
-    assert result_df.percentile[1] > result_df.percentile.iloc[-1]
+    assert result.enrichment_percentile[1] > result.enrichment_percentile.iloc[-1]
 
 
 def test_compare_functions():
-    percentile = 0.9
+    percent_list = [0.95, 0.9]
     eval_res = evaluate(
         profiles=df,
         features=features,
         meta_features=meta_features,
         replicate_groups=replicate_groups,
         operation="enrichment",
         similarity_metric="pearson",
-        enrichment_percentile=percentile,
+        enrichment_percentile=percent_list,
     )
     enr_res = enrichment(
         similarity_melted_df=similarity_melted_df,
         replicate_groups=replicate_groups,
-        percentile=percentile,
+        percentile=percent_list,
     )
-    assert enr_res == eval_res
+    assert enr_res.equals(eval_res)

cytominer_eval/tests/test_operations/test_precision_recall.py

@@ -42,11 +42,11 @@ def test_precision_recall():
     result = precision_recall(
         similarity_melted_df=similarity_melted_df,
         replicate_groups=replicate_groups,
-        k=10,
+        k_list=[5, 10],
     )
 
-    assert len(result.k.unique()) == 1
-    assert result.k.unique()[0] == 10
+    assert len(result.k.unique()) == 2
+    assert result.k.unique()[0] == 5
 
     # ITGAV has a really strong profile
     assert (

cytominer_eval/transform/util.py

@@ -7,7 +7,13 @@
 
 def get_available_eval_metrics():
     r"""Output the available eval metrics in the cytominer_eval library"""
-    return ["replicate_reproducibility", "precision_recall", "grit", "mp_value", "enrichment"]
+    return [
+        "replicate_reproducibility",
+        "precision_recall",
+        "grit",
+        "mp_value",
+        "enrichment",
+    ]
 
 
 def get_available_similarity_metrics():