diff --git a/infra/cnt_per_iters_plot.py b/infra/cnt_per_iters_plot.py new file mode 100644 index 0000000..7e1951c --- /dev/null +++ b/infra/cnt_per_iters_plot.py @@ -0,0 +1,47 @@ +import argparse +import json +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +def plot_cnt_per_iters(outcomes, args): + # Create figure + fig, ax = plt.subplots(figsize=(4, 3.5)) + fig.tight_layout(pad=2.0) + + # Drop precision column and sum up based on iteration + # outcomes = outcomes.drop(['baseline_precision'], axis=1) + # outcomes = outcomes.groupby(['rival_iter', 'tool_name'], as_index=False).sum() + + # Select tools + baseline = outcomes.loc[(outcomes['tool_name'] == "valid-baseline") & (outcomes['baseline_precision'] > 73)] + baseline = baseline.drop(['rival_iter'], axis=1) + baseline = baseline.groupby(['baseline_precision'], as_index=False).sum() + + rival = outcomes.loc[(outcomes['tool_name'] == "valid-rival") & (outcomes['rival_iter'] > 0)] + rival = rival.drop(['baseline_precision'], axis=1) + rival = rival.groupby(['rival_iter'], as_index=False).sum() + + ax.bar(np.arange(len(baseline)) + 0.9, baseline['number_of_points'], color="green", alpha=1, width=0.4, label='baseline', hatch='/') + ax.bar(np.arange(len(rival)) + 1.1, rival['number_of_points'], color="red", alpha=0.7, width=0.4, label='reval') + + ax.legend() + ax.set_xlabel("Iteration") + ax.set_ylabel("Count of converged points") + ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3) + plt.tight_layout() + plt.savefig(args.path + "/cnt_per_iters_plot.png", format="png") + plt.savefig(args.path + "/cnt_per_iters_plot.pdf", format="pdf") + +def load_outcomes(path): + outcomes = json.load(open(path, "r"))["outcomes"] + outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'baseline_precision', 'tool_name', 'number_of_points']) + return outcomes + +parser = argparse.ArgumentParser(prog='ratio_plot.py', description='Script outputs ratio plots') +parser.add_argument('-t', '--timeline', dest='timeline', default="report/timeline.json") +parser.add_argument('-o', '--output-path', dest='path', default="report") +args = parser.parse_args() + +outcomes = load_outcomes(args.timeline) +plot_cnt_per_iters(outcomes, args) diff --git a/infra/nightly.sh b/infra/nightly.sh index defe679..c2dc1ba 100644 --- a/infra/nightly.sh +++ b/infra/nightly.sh @@ -63,6 +63,7 @@ function perf { python3 infra/point_graph.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR" python3 infra/histograms.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR" python3 infra/cnt_per_iters_plot.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR" + python3 infra/repeats_plot.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR" cp profile.json "$REPORTDIR"/profile.json cp profile.js "$REPORTDIR"/profile.js } diff --git a/infra/repeats_plot.py b/infra/repeats_plot.py new file mode 100644 index 0000000..0ef5105 --- /dev/null +++ b/infra/repeats_plot.py @@ -0,0 +1,39 @@ +import argparse +import json +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +def plot_repeats_plot(outcomes, args): + # Create figure + fig, ax = plt.subplots(figsize=(4, 3.5)) + fig.tight_layout(pad=2.0) + + # Drop precision column and sum up based on iteration + rival = outcomes.loc[(outcomes['tool'] == "rival") & (outcomes['iter'] > 0)] + baseline = outcomes.loc[(outcomes['tool'] == "baseline") & (outcomes['iter'] > 0)] + + ax.bar(np.arange(len(baseline)) + 0.9, baseline['number_of_instr_executions'], color="green", alpha=1, width=0.4, label='baseline', hatch='/') + ax.bar(np.arange(len(rival)) + 1.1, rival['number_of_instr_executions'], color="red", alpha=0.7, width=0.4, label='reval') + + ax.legend() + ax.set_xlabel("Iteration") + ax.set_ylabel("Number of instructions executions") + ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3) + plt.tight_layout() + plt.savefig(args.path + "/repeats_plot.png", format="png") + plt.savefig(args.path + "/repeats_plot.pdf", format="pdf") + +def load_outcomes(path): + outcomes = json.load(open(path, "r"))["instr-executed-cnt"] + outcomes = pd.DataFrame(outcomes, columns=['tool', 'iter', 'number_of_instr_executions']) + return outcomes + +parser = argparse.ArgumentParser(prog='repeats_plot.py', description='Script outputs repeats plots') +parser.add_argument('-t', '--timeline', dest='timeline', default="report/timeline.json") +parser.add_argument('-o', '--output-path', dest='path', default="report") +args = parser.parse_args() + +outcomes = load_outcomes(args.timeline) +plot_repeats_plot(outcomes, args) + diff --git a/time.rkt b/time.rkt index 7ceb0cd..62e7838 100644 --- a/time.rkt +++ b/time.rkt @@ -90,6 +90,12 @@ 'mixsample-rival-all (list (execution-time execution) name precision))) + ; Record number of instructions has been executed + (when (equal? rival-status 'valid) + (timeline-push! timeline + 'instr-executed-cnt + (list 'rival rival-iter (vector-length rival-executions)))) + ; --------------------------- Baseline execution ---------------------------------------------- (define baseline-start-apply (current-inexact-milliseconds)) (match-define (list baseline-status baseline-exs) @@ -117,6 +123,11 @@ 'mixsample-baseline-all (list (execution-time execution) name precision))) + (when (equal? rival-status 'valid) + (timeline-push! timeline + 'instr-executed-cnt + (list 'baseline rival-iter (vector-length baseline-executions)))) + ; --------------------------- Sollya execution ------------------------------------------------ ; Points for expressions where Sollya has not compiled do not go to the plot/speed graphs! ; Also, if Rival's status is invalid - these points do not go to the graphs! @@ -222,6 +233,11 @@ (match-define (list time* name precision) args*) (define time (hash-ref mixsample-hash (list name precision) (λ () 0))) (hash-set! mixsample-hash (list name precision) (+ time time*))] + ['instr-executed-cnt + (define instr-cnt-hash (hash-ref timeline key)) + (match-define (list tool iter cnt) args*) + (define cnt* (hash-ref instr-cnt-hash (list tool iter) (λ () 0))) + (hash-set! instr-cnt-hash (list tool iter) (+ cnt cnt*))] [else (error "Unknown key for timeline!")])) (define (timeline->jsexpr timeline) @@ -239,7 +255,10 @@ (list value (car key) (second key))) 'mixsample-baseline-all (for/list ([(key value) (in-hash (hash-ref timeline 'mixsample-baseline-all))]) - (list value (car key) (second key))))) + (list value (car key) (second key))) + 'instr-executed-cnt + (for/list ([(key value) (in-hash (hash-ref timeline 'instr-executed-cnt))]) + (list (~a (car key)) (second key) value)))) (define (make-expression-table points test-id timeline-port) (newline) @@ -257,7 +276,8 @@ (cons 'mixsample-rival-valid (make-hash)) (cons 'mixsample-baseline-valid (make-hash)) (cons 'mixsample-rival-all (make-hash)) - (cons 'mixsample-baseline-all (make-hash))))) + (cons 'mixsample-baseline-all (make-hash)) + (cons 'instr-executed-cnt (make-hash))))) (define table (for/list ([rec (in-port read-json points)] @@ -404,6 +424,7 @@ (html-add-plot html-port "ratio_plot_precision.png") (html-add-plot html-port "point_graph.png") (html-add-plot html-port "cnt_per_iters_plot.png") + (html-add-plot html-port "repeats_plot.png") (html-add-histogram html-port "histogram_valid.png") (html-add-histogram html-port "histogram_all.png"))