From 86a25ece307d9144f3ea47ac8763e3b6500c526f Mon Sep 17 00:00:00 2001 From: yukirora Date: Tue, 13 Aug 2024 07:48:23 +0000 Subject: [PATCH 1/3] fix bug and warning in data diagnosis --- superbench/analyzer/data_analysis.py | 2 +- superbench/analyzer/data_diagnosis.py | 11 +++++------ superbench/analyzer/diagnosis_rule_op.py | 10 ++++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/superbench/analyzer/data_analysis.py b/superbench/analyzer/data_analysis.py index 5a7fb1ed8..5104051d4 100644 --- a/superbench/analyzer/data_analysis.py +++ b/superbench/analyzer/data_analysis.py @@ -243,7 +243,7 @@ def aggregate(raw_data_df, pattern=None): match = re.search(pattern, metric) if match: metric_in_list = list(metric) - for i in range(1, len(match.groups()) + 1): + for i in range(len(match.groups()) , 0, -1): metric_in_list[match.start(i):match.end(i)] = '*' short = ''.join(metric_in_list) if short not in metric_store: diff --git a/superbench/analyzer/data_diagnosis.py b/superbench/analyzer/data_diagnosis.py index ee5d705b6..b39a91e80 100644 --- a/superbench/analyzer/data_diagnosis.py +++ b/superbench/analyzer/data_diagnosis.py @@ -262,9 +262,8 @@ def output_all_nodes_results(self, raw_data_df, data_not_accept_df): all_data_df = data_not_accept_df[[ append_columns[index] ]].merge(all_data_df, left_index=True, right_index=True, how='right') - all_data_df['Accept'] = all_data_df['Accept'].replace(np.nan, True) - all_data_df['Number Of Issues'] = all_data_df['Number Of Issues'].replace(np.nan, 0) - all_data_df['Number Of Issues'] = all_data_df['Number Of Issues'].astype(int) + all_data_df['Accept'] = all_data_df['Accept'].replace(np.nan, 1).astype('bool') + all_data_df['Number Of Issues'] = all_data_df['Number Of Issues'].replace(np.nan, 0).astype('int') return all_data_df @@ -296,7 +295,7 @@ def output_diagnosis_in_jsonl(self, data_not_accept_df, output_path): data_not_accept_df (DataFrame): the DataFrame to output output_path (str): the path of output jsonl file """ - data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').fillna(self.na) + data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').infer_objects().fillna(self.na) p = Path(output_path) try: data_not_accept_json = data_not_accept_df.to_json(orient='index') @@ -327,7 +326,7 @@ def output_diagnosis_in_json(self, data_not_accept_df, output_path): data_not_accept_df (DataFrame): the DataFrame to output output_path (str): the path of output jsonl file """ - data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').fillna(self.na) + data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').infer_objects().fillna(self.na) data_not_accept_df = data_not_accept_df.reset_index() data_not_accept_df = data_not_accept_df.rename( columns={ @@ -378,7 +377,7 @@ def generate_md_lines(self, data_not_accept_df, rules, round): data_not_accept_df = data_analysis.round_significant_decimal_places( data_not_accept_df, round, [metric] ) - data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').fillna(self.na) + data_not_accept_df = data_not_accept_df.convert_dtypes().astype('object').infer_objects().fillna(self.na) lines = file_handler.generate_md_table(data_not_accept_df, header) return lines diff --git a/superbench/analyzer/diagnosis_rule_op.py b/superbench/analyzer/diagnosis_rule_op.py index c4aedd80f..04bad8ab7 100644 --- a/superbench/analyzer/diagnosis_rule_op.py +++ b/superbench/analyzer/diagnosis_rule_op.py @@ -239,17 +239,19 @@ def failure_check(data_row, rule, summary_data_row, details, categories, raw_rul violated_metric_num = 0 for metric_regex in raw_rule['metrics']: match = False + violate = False for metric in rule['metrics']: if re.search(metric_regex, metric): match = True # metric not in raw_data or the value is none, miss test - if metric not in data_row or pd.isna(data_row[metric]): - violated_metric_num += 1 - break + if RuleOp.miss_test(metric, rule, data_row, details, categories): + violate = True # metric_regex written in rules is not matched by any metric, miss test if not match: - violated_metric_num += 1 + violate = True RuleOp.add_categories_and_details(metric_regex + '_miss', rule['categories'], details, categories) + if violate: + violated_metric_num += 1 # return code != 0, failed test violated_metric_num += RuleOp.value(data_row, rule, summary_data_row, details, categories) return violated_metric_num From 6c58b1a8d19d75c52457f6c3d7b9ac2fe4366d75 Mon Sep 17 00:00:00 2001 From: yukirora Date: Tue, 13 Aug 2024 08:29:19 +0000 Subject: [PATCH 2/3] fix lint and test issue --- superbench/analyzer/data_analysis.py | 2 +- superbench/analyzer/diagnosis_rule_op.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/superbench/analyzer/data_analysis.py b/superbench/analyzer/data_analysis.py index 5104051d4..99895655c 100644 --- a/superbench/analyzer/data_analysis.py +++ b/superbench/analyzer/data_analysis.py @@ -243,7 +243,7 @@ def aggregate(raw_data_df, pattern=None): match = re.search(pattern, metric) if match: metric_in_list = list(metric) - for i in range(len(match.groups()) , 0, -1): + for i in range(len(match.groups()), 0, -1): metric_in_list[match.start(i):match.end(i)] = '*' short = ''.join(metric_in_list) if short not in metric_store: diff --git a/superbench/analyzer/diagnosis_rule_op.py b/superbench/analyzer/diagnosis_rule_op.py index 04bad8ab7..ff80e9049 100644 --- a/superbench/analyzer/diagnosis_rule_op.py +++ b/superbench/analyzer/diagnosis_rule_op.py @@ -254,6 +254,7 @@ def failure_check(data_row, rule, summary_data_row, details, categories, raw_rul violated_metric_num += 1 # return code != 0, failed test violated_metric_num += RuleOp.value(data_row, rule, summary_data_row, details, categories) + details[:] = list(dict.fromkeys(details)) # remove duplicate details return violated_metric_num From 4390bc29106e77ed4ce6c828de9289400e85e851 Mon Sep 17 00:00:00 2001 From: yukirora Date: Tue, 13 Aug 2024 08:40:38 +0000 Subject: [PATCH 3/3] fix lint issue --- superbench/analyzer/diagnosis_rule_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superbench/analyzer/diagnosis_rule_op.py b/superbench/analyzer/diagnosis_rule_op.py index ff80e9049..94595e8a1 100644 --- a/superbench/analyzer/diagnosis_rule_op.py +++ b/superbench/analyzer/diagnosis_rule_op.py @@ -254,7 +254,7 @@ def failure_check(data_row, rule, summary_data_row, details, categories, raw_rul violated_metric_num += 1 # return code != 0, failed test violated_metric_num += RuleOp.value(data_row, rule, summary_data_row, details, categories) - details[:] = list(dict.fromkeys(details)) # remove duplicate details + details[:] = list(dict.fromkeys(details)) # remove duplicate details return violated_metric_num