plot updates!

infra/cnt_per_iters_plot.py

@@ -0,0 +1,47 @@
+import argparse
+import json
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def plot_cnt_per_iters(outcomes, args):
+    # Create figure
+    fig, ax = plt.subplots(figsize=(4, 3.5))
+    fig.tight_layout(pad=2.0)
+
+    # Drop precision column and sum up based on iteration
+    # outcomes = outcomes.drop(['baseline_precision'], axis=1)
+    # outcomes = outcomes.groupby(['rival_iter', 'tool_name'], as_index=False).sum()
+
+    # Select tools
+    baseline = outcomes.loc[(outcomes['tool_name'] == "valid-baseline") & (outcomes['baseline_precision'] > 73)]
+    baseline = baseline.drop(['rival_iter'], axis=1)
+    baseline = baseline.groupby(['baseline_precision'], as_index=False).sum()
+
+    rival = outcomes.loc[(outcomes['tool_name'] == "valid-rival") & (outcomes['rival_iter'] > 0)]
+    rival = rival.drop(['baseline_precision'], axis=1)
+    rival = rival.groupby(['rival_iter'], as_index=False).sum()
+
+    ax.bar(np.arange(len(baseline)) + 0.9, baseline['number_of_points'], color="green", alpha=1, width=0.4, label='baseline', hatch='/')
+    ax.bar(np.arange(len(rival)) + 1.1, rival['number_of_points'], color="red", alpha=0.7, width=0.4, label='reval')
+
+    ax.legend()
+    ax.set_xlabel("Iteration")
+    ax.set_ylabel("Count of converged points")
+    ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3)
+    plt.tight_layout()
+    plt.savefig(args.path + "/cnt_per_iters_plot.png", format="png")
+    plt.savefig(args.path + "/cnt_per_iters_plot.pdf", format="pdf")
+
+def load_outcomes(path):
+    outcomes = json.load(open(path, "r"))["outcomes"]
+    outcomes = pd.DataFrame(outcomes, columns=['time', 'rival_iter', 'baseline_precision', 'tool_name', 'number_of_points'])
+    return outcomes
+
+parser = argparse.ArgumentParser(prog='ratio_plot.py', description='Script outputs ratio plots')
+parser.add_argument('-t', '--timeline', dest='timeline', default="report/timeline.json")
+parser.add_argument('-o', '--output-path', dest='path', default="report")
+args = parser.parse_args()
+
+outcomes = load_outcomes(args.timeline)
+plot_cnt_per_iters(outcomes, args)

infra/nightly.sh

@@ -63,6 +63,7 @@ function perf {
     python3 infra/point_graph.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR"
     python3 infra/histograms.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR"
     python3 infra/cnt_per_iters_plot.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR"
+    python3 infra/repeats_plot.py -t "$REPORTDIR"/timeline.json -o "$REPORTDIR"
     cp profile.json "$REPORTDIR"/profile.json
     cp profile.js "$REPORTDIR"/profile.js
 }

infra/repeats_plot.py

@@ -0,0 +1,39 @@
+import argparse
+import json
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+def plot_repeats_plot(outcomes, args):
+    # Create figure
+    fig, ax = plt.subplots(figsize=(4, 3.5))
+    fig.tight_layout(pad=2.0)
+
+    # Drop precision column and sum up based on iteration
+    rival = outcomes.loc[(outcomes['tool'] == "rival") & (outcomes['iter'] > 0)]
+    baseline = outcomes.loc[(outcomes['tool'] == "baseline") & (outcomes['iter'] > 0)]
+
+    ax.bar(np.arange(len(baseline)) + 0.9, baseline['number_of_instr_executions'], color="green", alpha=1, width=0.4, label='baseline', hatch='/')
+    ax.bar(np.arange(len(rival)) + 1.1, rival['number_of_instr_executions'], color="red", alpha=0.7, width=0.4, label='reval')
+
+    ax.legend()
+    ax.set_xlabel("Iteration")
+    ax.set_ylabel("Number of instructions executions")
+    ax.yaxis.grid(True, linestyle='-', which='major', color='grey', alpha=0.3)
+    plt.tight_layout()
+    plt.savefig(args.path + "/repeats_plot.png", format="png")
+    plt.savefig(args.path + "/repeats_plot.pdf", format="pdf")
+
+def load_outcomes(path):
+    outcomes = json.load(open(path, "r"))["instr-executed-cnt"]
+    outcomes = pd.DataFrame(outcomes, columns=['tool', 'iter', 'number_of_instr_executions'])
+    return outcomes
+
+parser = argparse.ArgumentParser(prog='repeats_plot.py', description='Script outputs repeats plots')
+parser.add_argument('-t', '--timeline', dest='timeline', default="report/timeline.json")
+parser.add_argument('-o', '--output-path', dest='path', default="report")
+args = parser.parse_args()
+
+outcomes = load_outcomes(args.timeline)
+plot_repeats_plot(outcomes, args)
+

time.rkt

@@ -90,6 +90,12 @@
                         'mixsample-rival-all
                         (list (execution-time execution) name precision)))
 
+      ; Record number of instructions has been executed
+      (when (equal? rival-status 'valid)
+        (timeline-push! timeline
+                        'instr-executed-cnt
+                        (list 'rival rival-iter (vector-length rival-executions))))
+
       ; --------------------------- Baseline execution ----------------------------------------------
       (define baseline-start-apply (current-inexact-milliseconds))
       (match-define (list baseline-status baseline-exs)
@@ -117,6 +123,11 @@
                         'mixsample-baseline-all
                         (list (execution-time execution) name precision)))
 
+      (when (equal? rival-status 'valid)
+        (timeline-push! timeline
+                        'instr-executed-cnt
+                        (list 'baseline rival-iter (vector-length baseline-executions))))
+
       ; --------------------------- Sollya execution ------------------------------------------------
       ; Points for expressions where Sollya has not compiled do not go to the plot/speed graphs!
       ; Also, if Rival's status is invalid - these points do not go to the graphs!
@@ -222,6 +233,11 @@
      (match-define (list time* name precision) args*)
      (define time (hash-ref mixsample-hash (list name precision) (λ () 0)))
      (hash-set! mixsample-hash (list name precision) (+ time time*))]
+    ['instr-executed-cnt
+     (define instr-cnt-hash (hash-ref timeline key))
+     (match-define (list tool iter cnt) args*)
+     (define cnt* (hash-ref instr-cnt-hash (list tool iter) (λ () 0)))
+     (hash-set! instr-cnt-hash (list tool iter) (+ cnt cnt*))]
     [else (error "Unknown key for timeline!")]))
 
 (define (timeline->jsexpr timeline)
@@ -239,7 +255,10 @@
           (list value (car key) (second key)))
         'mixsample-baseline-all
         (for/list ([(key value) (in-hash (hash-ref timeline 'mixsample-baseline-all))])
-          (list value (car key) (second key)))))
+          (list value (car key) (second key)))
+        'instr-executed-cnt
+        (for/list ([(key value) (in-hash (hash-ref timeline 'instr-executed-cnt))])
+          (list (~a (car key)) (second key) value))))
 
 (define (make-expression-table points test-id timeline-port)
   (newline)
@@ -257,7 +276,8 @@
            (cons 'mixsample-rival-valid (make-hash))
            (cons 'mixsample-baseline-valid (make-hash))
            (cons 'mixsample-rival-all (make-hash))
-           (cons 'mixsample-baseline-all (make-hash)))))
+           (cons 'mixsample-baseline-all (make-hash))
+           (cons 'instr-executed-cnt (make-hash)))))
 
   (define table
     (for/list ([rec (in-port read-json points)]
@@ -404,6 +424,7 @@
     (html-add-plot html-port "ratio_plot_precision.png")
     (html-add-plot html-port "point_graph.png")
     (html-add-plot html-port "cnt_per_iters_plot.png")
+    (html-add-plot html-port "repeats_plot.png")
     (html-add-histogram html-port "histogram_valid.png")
     (html-add-histogram html-port "histogram_all.png"))