initial sketch of benchmarking

SignalAnalysisBenchmarks/Project.toml

@@ -0,0 +1,4 @@
+name='SignalAnalysisBenchmarks'
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+SignalAnalysis = "df1fea92-c066-49dd-8b36-eace3378ea47"

SignalAnalysisBenchmarks/makefile

@@ -0,0 +1,2 @@
+test:
+	./run.sh

SignalAnalysisBenchmarks/run.sh

@@ -0,0 +1,17 @@
+#!/bin/zsh
+MOD=SignalAnalysisBenchmarks
+
+DT=$(date '+%Y-%m-%d_%H:%M:%S')
+TMP=/tmp/SignalAnalysis
+[ -d $TMP ] || mkdir -p $TMP
+FNAME="$TMP/regressions_$DT.json"
+SCRIPT=$(cat <<EOF
+using $MOD 
+using BenchmarkTools
+$MOD.load!("dsp";tune=false)
+@info "running benchmark suite..."
+results = run($MOD.SUITE["dsp"])
+BenchmarkTools.save("$FNAME", results)
+EOF
+)
+julia --project=. -e $SCRIPT --color=no

SignalAnalysisBenchmarks/src/SignalAnalysisBenchmarks.jl

@@ -0,0 +1,52 @@
+module SignalAnalysisBenchmarks
+using BenchmarkTools
+using SignalAnalysis
+
+BenchmarkTools.DEFAULT_PARAMETERS.seconds = 1.0
+BenchmarkTools.DEFAULT_PARAMETERS.samples = 10000
+BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = 0.15
+BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = 0.01
+
+const PARAMS_PATH = joinpath(dirname(@__FILE__), "..", "etc", "params.json")
+const SUITE = BenchmarkGroup()
+
+    const MODULES = Dict( 
+        "dsp" => :DspBenchmarks
+    )
+
+load!(id::AbstractString; kwargs...) = load!(SUITE, id; kwargs...)
+
+function load!(group::BenchmarkGroup, id::AbstractString; tune::Bool = true)
+    modsym = MODULES[id]
+    modpath = joinpath(dirname(@__FILE__), id, "$(modsym).jl")
+    Core.eval(SignalAnalysisBenchmarks, :(include($modpath)))
+    modsuite = Core.eval(SignalAnalysisBenchmarks, modsym).SUITE
+    group[id] = modsuite
+    if tune
+        results = BenchmarkTools.load(PARAMS_PATH)[1]
+        haskey(results, id) && loadparams!(modsuite, results[id], :evals)
+    end
+    return group
+end
+
+#TODO: add tuning
+function loadall!(group::BenchmarkGroup; verbose::Bool = true, tune::Bool = false)
+    for id in keys(MODULES)
+        if verbose
+            print("loading group $(repr(id))... ")
+            time = @elapsed load!(group, id, tune=false)
+            println("done (took $time seconds)")
+        else
+            load!(group, id, tune=false)
+        end
+    end
+    if tune
+        results = BenchmarkTools.load(PARAMS_PATH)[1]
+        for (id, suite) in group
+            haskey(results, id) && loadparams!(suite, results[id], :evals)
+        end
+    end
+    return group
+end
+
+end

SignalAnalysisBenchmarks/src/dsp/DspBenchmarks.jl

@@ -0,0 +1,26 @@
+module DspBenchmarks
+    using SignalAnalysis: goertzel
+    using BenchmarkTools
+
+    const SUITE = BenchmarkGroup()
+    SUITE["goertzel"] = BenchmarkGroup(["math","dsp"])
+    # function goertzel(x::AbstractVector, f, n; fs=framerate(x))
+    #   function goertzel(x::AbstractVector, f; fs=framerate(x))
+    #   function goertzel(x::AbstractMatrix, f, n; fs=framerate(x))k
+    goertzel_containers = [Vector,Matrix];
+    datatypes = [Float64];
+    #can feed a vector with a block size or not
+    Fs = 34100;
+    dims = [(Fs* duration,n_sensors) for n_sensors in 2:2:10 for duration in 1:2:10];
+    vector_sizes = (1:2:10) .* Fs
+    ns = [13 14 15].^2;
+    Ts =[Float64,Float32,Float64];
+    for n in ns
+        for vector_size in vector_sizes
+            for T ∈ Ts 
+                V = rand(T,vector_size)
+                SUITE["goertzel_vector"][string(vector_size),string(eltype(T))] = @benchmarkable goertzel($V,2200,$n;fs = $Fs)
+            end
+        end 
+    end
+end