diff --git a/libs/core/testing/src/performance.cpp b/libs/core/testing/src/performance.cpp index 95caefcba7c..73bc6e76c70 100644 --- a/libs/core/testing/src/performance.cpp +++ b/libs/core/testing/src/performance.cpp @@ -181,10 +181,13 @@ average: {{average(elapsed)}}{{^-last}} << "\n\n"; } if (print_cdash_img) - strm << "" - << "./" << test_name_ - << ".png\n"; + { + for (std::size_t i = 0; i < obj.m_map.size(); i++) + strm << "" + << "./" << test_name_ << "_" << i + << ".png\n"; + } } return strm; } @@ -222,9 +225,12 @@ average: {{average(elapsed)}}{{^-last}} { detail::bench().render(templ, strm); if (!detailed_ && print_cdash_img) - strm << "" - << "./" << test_name_ << ".png\n"; + { + for (long unsigned int i = 0; i < detail::bench().results().size(); i++) + strm << "" + << "./" << test_name_ << "_" << i << ".png\n"; + } } // Overload that uses a default nanobench template diff --git a/tools/perftests_plot.py b/tools/perftests_plot.py index 3b043661607..cee33d96096 100755 --- a/tools/perftests_plot.py +++ b/tools/perftests_plot.py @@ -9,10 +9,10 @@ sns.set_style("ticks",{'axes.grid' : True}) -def mean_statistic(sample1, sample2, axis=-1): - mean1 = np.mean(sample1, axis=axis) - mean2 = np.mean(sample2, axis=axis) - return (mean1 - mean2) / mean1 +def median_statistic(sample1, sample2, axis=-1): + median1 = np.median(sample1, axis=axis) + median2 = np.median(sample2, axis=axis) + return (median2 - median1) / median1 rng = np.random.default_rng() @@ -33,9 +33,9 @@ def mean_statistic(sample1, sample2, axis=-1): samples = [] header_flag = True - n = ceil(len(json_obj1["outputs"]) / 2) - fig, ax = plt.subplots(2, n, figsize=(5 * n, 6), sharey=False) - plt.subplots_adjust(hspace=0.3) + # n = ceil(len(json_obj1["outputs"]) / 2) + # fig, ax = plt.subplots(2, n, figsize=(5 * n, 6), sharey=False) + # plt.subplots_adjust(hspace=0.3) i = 0 for test1, test2 in zip(json_obj1["outputs"], json_obj2["outputs"]): if test1["name"] == test2["name"]: @@ -48,22 +48,27 @@ def mean_statistic(sample1, sample2, axis=-1): samples.append(test1["series"]) data = (test2["series"], test1["series"]) - res = scipy.stats.bootstrap(data, mean_statistic, method='basic', random_state=rng) + res = scipy.stats.bootstrap(data, median_statistic, method='basic', random_state=rng) mean2 = np.mean(test2["series"]) mean1 = np.mean(test1["series"]) - if n != 1: - curr_plot = ax[i // n, i % n] - else: - curr_plot = ax[i] + # if n != 1: + # curr_plot = ax[i // n, i % n] + # else: + # curr_plot = ax[i] - sns.kdeplot(test2["series"], fill=True, ax=curr_plot, label='baseline') - sns.kdeplot(test1["series"], fill=True, ax=curr_plot, label='current') - curr_plot.axvline(mean2, label='baseline mean', color='k') - curr_plot.axvline(mean1, label='current mean', color='g') - curr_plot.legend() - curr_plot.set_title(f'{test1["name"]}, {test1["executor"]}') + plt.figure(figsize=(8, 4)) + + sns.kdeplot(test2["series"], fill=True, label='baseline') + sns.kdeplot(test1["series"], fill=True, label='current') + plt.axvline(mean2, label='baseline mean', color='k') + plt.axvline(mean1, label='current mean', color='g') + plt.legend() + plt.suptitle(f'{test1["name"]}, \n{test1["executor"]}') + + plt.tight_layout() + plt.savefig(f"{sys.argv[3]}_{i}.png") percentage_diff = ((mean2 - mean1) / mean2) * 100 @@ -88,7 +93,7 @@ def mean_statistic(sample1, sample2, axis=-1): html_file.close() - plt.tight_layout() - [fig.delaxes(a) for a in ax.flatten() if not a.has_data()] - plt.savefig(sys.argv[3] + ".png") + # plt.tight_layout() + # [fig.delaxes(a) for a in ax.flatten() if not a.has_data()] + # plt.savefig(sys.argv[3] + ".png") \ No newline at end of file