add flapping check in client wait

quintilesims · Dec 8, 2016 · eee12b2 · eee12b2
1 parent 793b528
commit eee12b2
Show file tree

Hide file tree

Showing 21 changed files with 108 additions and 27 deletions.
diff --git a/cli/client/service.go b/cli/client/service.go
@@ -7,6 +7,8 @@ import (
 	"time"
 )
 
+const REQUIRED_SUCCESS_WAIT_COUNT = 3
+
 func (c *APIClient) CreateService(name, environmentID, deployID, serviceID string) (*models.Service, error) {
 	req := models.CreateServiceRequest{
 		ServiceName:    name,
@@ -91,6 +93,8 @@ func (c *APIClient) ScaleService(id string, count int) (*models.Service, error)
 }
 
 func (c *APIClient) WaitForDeployment(serviceID string, timeout time.Duration) (*models.Service, error) {
+	var successCount int
+
 	waiter := waitutils.Waiter{
 		Name:    "WaitForDeployment",
 		Timeout: timeout,
@@ -108,7 +112,8 @@ func (c *APIClient) WaitForDeployment(serviceID string, timeout time.Duration) (
 				}
 			}
 
-			return true, nil
+			successCount++
+			return successCount >= REQUIRED_SUCCESS_WAIT_COUNT, nil
 		},
 	}
 

diff --git a/cli/client/service_test.go b/cli/client/service_test.go
@@ -5,6 +5,7 @@ import (
 	"github.com/quintilesims/layer0/common/testutils"
 	"net/http"
 	"testing"
+	"time"
 )
 
 func TestCreateService(t *testing.T) {
@@ -177,3 +178,64 @@ func TestUpdateService(t *testing.T) {
 
 	testutils.AssertEqual(t, service.ServiceID, "id")
 }
+
+func TestWaitForDeployment(t *testing.T) {
+	var count int
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		count++
+		testutils.AssertEqual(t, r.Method, "GET")
+		testutils.AssertEqual(t, r.URL.Path, "/service/id")
+
+		var runningCount int64 = 1
+		var desiredCount int64 = 2
+
+		// simulate flapping success
+		if count == 0  || count > 3 {
+			runningCount = 2
+		}
+
+		service := models.Service{
+			ServiceID: "id",
+			Deployments: []models.Deployment{
+				{DesiredCount: desiredCount, RunningCount: runningCount},
+				{DesiredCount: desiredCount, RunningCount: runningCount},
+			},
+		}
+
+		MarshalAndWrite(t, w, service, 200)
+	}
+
+	client, server := newClientAndServer(handler)
+	defer server.Close()
+
+	service, err := client.WaitForDeployment("id", time.Minute*15)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testutils.AssertEqual(t, service.ServiceID, "id")
+	if count <  REQUIRED_SUCCESS_WAIT_COUNT {
+		t.Fatalf("Retry count was less than required (%d)", count)
+	}
+}
+
+func TestWaitForDeployment_timeout(t *testing.T) {
+        handler := func(w http.ResponseWriter, r *http.Request) {
+                 service := models.Service{
+                        Deployments: []models.Deployment{
+                                {DesiredCount: 1, RunningCount: 0},
+                        },
+                }
+
+		MarshalAndWrite(t, w, service, 200)
+        }
+
+        client, server := newClientAndServer(handler)
+        defer server.Close()
+
+        if _, err := client.WaitForDeployment("id", time.Millisecond); err == nil {
+                t.Fatal("Error was nil!")
+        }
+}
+
diff --git a/cli/command/admin_command.go b/cli/command/admin_command.go
@@ -1,8 +1,8 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/config"
+	"github.com/urfave/cli"
 )
 
 type AdminCommand struct {

diff --git a/cli/command/certificate_command.go b/cli/command/certificate_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"io/ioutil"
 )
 

diff --git a/cli/command/command.go b/cli/command/command.go
@@ -1,10 +1,10 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/client"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/cli/printer"
+	"github.com/urfave/cli"
 )
 
 type CommandGroup interface {

diff --git a/cli/command/command_test.go b/cli/command/command_test.go
@@ -4,10 +4,10 @@ import (
 	"flag"
 	"fmt"
 	"github.com/golang/mock/gomock"
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/client/mock_client"
 	"github.com/quintilesims/layer0/cli/command/mock_command"
 	"github.com/quintilesims/layer0/cli/printer"
+	"github.com/urfave/cli"
 	"io/ioutil"
 	"os"
 	"testing"

diff --git a/cli/command/deploy_command.go b/cli/command/deploy_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"io/ioutil"
 )
 

diff --git a/cli/command/deploy_command_test.go b/cli/command/deploy_command_test.go
@@ -1,8 +1,8 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/command/environment_command.go b/cli/command/environment_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"io/ioutil"
 	"strconv"
 )

diff --git a/cli/command/environment_command_test.go b/cli/command/environment_command_test.go
@@ -1,8 +1,8 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/command/job_command.go b/cli/command/job_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 )
 
 type JobCommand struct {

diff --git a/cli/command/job_command_test.go b/cli/command/job_command_test.go
@@ -1,8 +1,8 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/command/load_balancer_command.go b/cli/command/load_balancer_command.go
@@ -2,9 +2,9 @@ package command
 
 import (
 	"fmt"
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"strconv"
 	"strings"
 )

diff --git a/cli/command/load_balancer_command_test.go b/cli/command/load_balancer_command_test.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
 	"github.com/quintilesims/layer0/common/testutils"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/command/service_command.go b/cli/command/service_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"strconv"
 )
 

diff --git a/cli/command/service_command_test.go b/cli/command/service_command_test.go
@@ -1,8 +1,8 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/command/task_command.go b/cli/command/task_command.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/urfave/cli"
 	"strings"
 )
 

diff --git a/cli/command/task_command_test.go b/cli/command/task_command_test.go
@@ -1,9 +1,9 @@
 package command
 
 import (
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/common/models"
 	"github.com/quintilesims/layer0/common/testutils"
+	"github.com/urfave/cli"
 	"testing"
 )
 

diff --git a/cli/main.go b/cli/main.go
@@ -3,12 +3,12 @@ package main
 import (
 	"fmt"
 	log "github.com/Sirupsen/logrus"
-	"github.com/urfave/cli"
 	"github.com/quintilesims/layer0/cli/client"
 	"github.com/quintilesims/layer0/cli/command"
 	"github.com/quintilesims/layer0/cli/printer"
 	"github.com/quintilesims/layer0/common/config"
 	"github.com/quintilesims/layer0/common/waitutils"
+	"github.com/urfave/cli"
 	"os"
 	"sync"
 	"time"

diff --git a/cli/printer/text.go b/cli/printer/text.go
@@ -3,10 +3,10 @@ package printer
 import (
 	"fmt"
 	"github.com/briandowns/spinner"
-	"github.com/ryanuber/columnize"
 	"github.com/quintilesims/layer0/cli/entity"
 	"github.com/quintilesims/layer0/cli/printer/table"
 	"github.com/quintilesims/layer0/common/models"
+	"github.com/ryanuber/columnize"
 	"os"
 	"time"
 )

diff --git a/common/testutils/stubclock.go b/common/testutils/stubclock.go
@@ -2,21 +2,35 @@ package testutils
 
 import (
 	"time"
+	"sync"
 )
 
 type StubClock struct {
-	InnerTime time.Time
+	Time time.Time
+	once sync.Once
 }
 
-func (this *StubClock) Now() time.Time {
-	this.InnerTime = this.InnerTime.Add(time.Millisecond * 20)
-	return this.InnerTime
+func (s *StubClock) init(){
+	if s.Time.IsZero(){
+		s.Time = time.Now()
+	}
 }
 
-func (this *StubClock) Sleep(s time.Duration) {
-	this.InnerTime = this.InnerTime.Add(s)
+func (s *StubClock) Now() time.Time {
+	s.once.Do(s.init)
+
+	s.Time = s.Time.Add(time.Millisecond * 20)
+	return s.Time
 }
 
-func (this *StubClock) Since(t time.Time) time.Duration {
-	return this.Now().Sub(t)
+func (s *StubClock) Sleep(d time.Duration) {
+	s.once.Do(s.init)
+
+	s.Time = s.Time.Add(d)
+}
+
+func (s *StubClock) Since(t time.Time) time.Duration {
+	s.once.Do(s.init)
+
+	return s.Now().Sub(t)
 }