From d828f7e35878eceace322e6b9183abc8d9cb69ac Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 10:21:21 -0700 Subject: [PATCH 1/7] New PushFixed function to add data points to fixed size arrays Instead of push simply appending data points to expanding Go slices; this change will allow a rolling window dataset to be analyzed for anomalous signals. --- anomalyze.go | 8 ++++++++ anomalyze_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/anomalyze.go b/anomalyze.go index bb3d225..6cd3802 100644 --- a/anomalyze.go +++ b/anomalyze.go @@ -139,6 +139,14 @@ func (a *Anomalyzer) Push(x float64) float64 { return a.Eval() } +func (a *Anomalyzer) PushFixed(x float64) float64 { + // Add data to fixed size array which will not grow + a.Data.PushFixed(x) + + // evaluate the anomalous probability + return a.Eval() +} + // Return the weighted average of all statistical tests // for anomaly detection, which yields the probability that // the currently observed behavior is anomalous. diff --git a/anomalyze_test.go b/anomalyze_test.go index 6d53215..13a9250 100644 --- a/anomalyze_test.go +++ b/anomalyze_test.go @@ -45,6 +45,30 @@ func TestAnomalyzer(t *testing.T) { assert.Tf(t, prob > 0.5, "Anomalyzer returned a probability that was too small") } +func TestAnomalyzerPushFixed(t *testing.T) { + conf := &AnomalyzerConf{ + Sensitivity: 0.1, + UpperBound: 5, + LowerBound: 0, + ActiveSize: 1, + NSeasons: 4, + Methods: []string{"cdf", "fence", "highrank", "lowrank", "magnitude"}, + } + + // initialize with empty data or an actual slice of floats + data := []float64{0.1, 2.05, 1.5, 2.5, 2.6, 2.55} + + anomalyzer, err := NewAnomalyzer(conf, data) + assert.Equal(t, nil, err, "Error initializing new anomalyzer") + + prob := anomalyzer.PushFixed(8.0) + prob = anomalyzer.PushFixed(10.0) + prob = anomalyzer.PushFixed(8.0) + prob = anomalyzer.PushFixed(9.0) + assert.Tf(t, prob > 0.5, "Anomalyzer returned a probability that was too small") + assert.Equal(t, len(anomalyzer.Data), 6, "Array size did not stay at original size") +} + func Example() { conf := &AnomalyzerConf{ Sensitivity: 0.1, From 416599f032d2ab5293aa52074641249edc375377 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 11:23:16 -0700 Subject: [PATCH 2/7] Plumbing through change to GoVectors PushFixed to handle returned error --- anomalyze.go | 9 ++++++--- anomalyze_test.go | 9 +++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/anomalyze.go b/anomalyze.go index 6cd3802..3e98c62 100644 --- a/anomalyze.go +++ b/anomalyze.go @@ -139,12 +139,15 @@ func (a *Anomalyzer) Push(x float64) float64 { return a.Eval() } -func (a *Anomalyzer) PushFixed(x float64) float64 { +func (a *Anomalyzer) PushFixed(x float64) (float64, error) { // Add data to fixed size array which will not grow - a.Data.PushFixed(x) + err := a.Data.PushFixed(x) + if err != nil { + return NA, err + } // evaluate the anomalous probability - return a.Eval() + return a.Eval(), nil } // Return the weighted average of all statistical tests diff --git a/anomalyze_test.go b/anomalyze_test.go index 13a9250..035a05e 100644 --- a/anomalyze_test.go +++ b/anomalyze_test.go @@ -61,10 +61,11 @@ func TestAnomalyzerPushFixed(t *testing.T) { anomalyzer, err := NewAnomalyzer(conf, data) assert.Equal(t, nil, err, "Error initializing new anomalyzer") - prob := anomalyzer.PushFixed(8.0) - prob = anomalyzer.PushFixed(10.0) - prob = anomalyzer.PushFixed(8.0) - prob = anomalyzer.PushFixed(9.0) + prob, err := anomalyzer.PushFixed(8.0) + prob, err = anomalyzer.PushFixed(10.0) + prob, err = anomalyzer.PushFixed(8.0) + prob, err = anomalyzer.PushFixed(9.0) + assert.Equal(t, err, nil, "There was an error with array size") assert.Tf(t, prob > 0.5, "Anomalyzer returned a probability that was too small") assert.Equal(t, len(anomalyzer.Data), 6, "Array size did not stay at original size") } From 1dae04f4f6a5f26c4b067722d31ca28084941be5 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 11:35:35 -0700 Subject: [PATCH 3/7] Minor clarity update to README for the NA const --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3cd8b6..0f335d2 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ To capture seasonality, the bootstrap ks test should consider an active window l ### Fence -The fence test can be configured to use custom `UpperBound` and `LowerBound` values for the fences. If no lower bound is desired, set the value of `LowerBound` to `anomalyzer.NA`. +The fence test can be configured to use custom `UpperBound` and `LowerBound` values for the fences. If no lower bound is desired, set the value of `LowerBound` to the const variable: `anomalyzer.NA`. ### Diff & Rank From 7349d990ed88c68eaf22afb64904cad73335abd3 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 12:05:00 -0700 Subject: [PATCH 4/7] Updating readme and code with warnings about mixing Push and FixedPush Mixing Push and FixedPush will result in errors, tests prove this problem(not committed yet). I'll leave that for another day when there's free time. --- README.md | 8 ++++++++ anomalyze.go | 1 + 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 0f335d2..451a3cc 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,14 @@ func main() { // by a call to the Eval method. prob := anom.Push(8.0) fmt.Println("Anomalous Probability:", prob) + + // PushFixed method will keep the size of the Data vector constant. + // Oldest data points will be evicted as points are added. + // WARNING: Mixing Push() and PushFixed() will result in failure! + anom2, _ := anomalyzer.NewAnomalyzer(conf, data) + prob2, _ := anom2.PushFixed(8.0) + // returns an error as second value if the array size changed unexpectantly + fmt.Println("Anomalous Probability:", prob2) } ``` diff --git a/anomalyze.go b/anomalyze.go index 3e98c62..06ba0fa 100644 --- a/anomalyze.go +++ b/anomalyze.go @@ -139,6 +139,7 @@ func (a *Anomalyzer) Push(x float64) float64 { return a.Eval() } +// WARNING: Mixing Push() and PushFixed() will result in failure! func (a *Anomalyzer) PushFixed(x float64) (float64, error) { // Add data to fixed size array which will not grow err := a.Data.PushFixed(x) From 4600a1163d667a0f19977387887a32da7679586a Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 12:13:40 -0700 Subject: [PATCH 5/7] Fixing spacing typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 451a3cc..4f75f0d 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ func main() { // PushFixed method will keep the size of the Data vector constant. // Oldest data points will be evicted as points are added. // WARNING: Mixing Push() and PushFixed() will result in failure! - anom2, _ := anomalyzer.NewAnomalyzer(conf, data) + anom2, _ := anomalyzer.NewAnomalyzer(conf, data) prob2, _ := anom2.PushFixed(8.0) // returns an error as second value if the array size changed unexpectantly fmt.Println("Anomalous Probability:", prob2) From 7a58565ef790b6f483d3075f21872c9cddb9e66a Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Fri, 30 Oct 2015 14:32:31 -0700 Subject: [PATCH 6/7] Tests to prove GoVector's: `pushfixed-handles-exception` branch changes Changed the PushFixed array code to not break if the array was expanded by an external call to the standard Push function. The two functions can now be called in a mixed fashion, and PushFixed will still attempt to use as little memory as possible. --- anomalyze_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/anomalyze_test.go b/anomalyze_test.go index 035a05e..e87ab4a 100644 --- a/anomalyze_test.go +++ b/anomalyze_test.go @@ -70,6 +70,33 @@ func TestAnomalyzerPushFixed(t *testing.T) { assert.Equal(t, len(anomalyzer.Data), 6, "Array size did not stay at original size") } +func TestAnomalyzerPushMixed(t *testing.T) { + conf := &AnomalyzerConf{ + Sensitivity: 0.1, + UpperBound: 5, + LowerBound: 0, + ActiveSize: 1, + NSeasons: 4, + Methods: []string{"cdf", "fence", "highrank", "lowrank", "magnitude"}, + } + + // initialize with empty data or an actual slice of floats + data := []float64{0.1, 2.05, 1.5, 2.5, 2.6, 2.55} + + anomalyzer, err := NewAnomalyzer(conf, data) + assert.Equal(t, nil, err, "Error initializing new anomalyzer") + + prob, err := anomalyzer.PushFixed(8.0) + prob = anomalyzer.Push(10.0) + prob, err = anomalyzer.PushFixed(8.0) + prob = anomalyzer.Push(9.0) + assert.Equal(t, err, nil, "There was an error with mixing array extension") + assert.Tf(t, prob > 0.5, "Anomalyzer returned a probability that was too small") + assert.Equal(t, len(anomalyzer.Data), 8, "Array size Push* functions failed to grow Data to expected size") + assert.Equal(t, anomalyzer.Data[7], 9.0) + assert.Equal(t, anomalyzer.Data[0], 1.5, "Two values were appended, two values were popped from the array. 3rd original element should be tail.") +} + func Example() { conf := &AnomalyzerConf{ Sensitivity: 0.1, From 5ea889d150f7a173eac8d243a7e20a893ad53d16 Mon Sep 17 00:00:00 2001 From: Josh Roppo Date: Tue, 10 Nov 2015 16:20:49 -0800 Subject: [PATCH 7/7] Removing warning for problem which has been fixed and tab fix --- README.md | 5 ++--- anomalyze.go | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f75f0d..c79302d 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,9 @@ func main() { // PushFixed method will keep the size of the Data vector constant. // Oldest data points will be evicted as points are added. - // WARNING: Mixing Push() and PushFixed() will result in failure! anom2, _ := anomalyzer.NewAnomalyzer(conf, data) - prob2, _ := anom2.PushFixed(8.0) - // returns an error as second value if the array size changed unexpectantly + prob2, _ := anom2.PushFixed(8.0) + // returns an error as second value if the array size changed unexpectantly fmt.Println("Anomalous Probability:", prob2) } ``` diff --git a/anomalyze.go b/anomalyze.go index 06ba0fa..3e98c62 100644 --- a/anomalyze.go +++ b/anomalyze.go @@ -139,7 +139,6 @@ func (a *Anomalyzer) Push(x float64) float64 { return a.Eval() } -// WARNING: Mixing Push() and PushFixed() will result in failure! func (a *Anomalyzer) PushFixed(x float64) (float64, error) { // Add data to fixed size array which will not grow err := a.Data.PushFixed(x)