Retry requests (#19)

* add "atoms" dependency
* retry
* export the variables to 
* mock the test IAMClientSpec
* update the version number 0.2.2 for future release
* update the read version
* correct README.md

README.md

@@ -18,7 +18,7 @@ Clients of this library don't need to change their code in order to protect endp
 Configure libraries dependencies in your `build.sbt`:
 
 ```scala
-libraryDependencies += "org.zalando" %% "play-zhewbacca" % "0.2.1"
+libraryDependencies += "org.zalando" %% "play-zhewbacca" % "0.2.2"
 ```
 
 To configure Development environment:
@@ -92,12 +92,18 @@ authorisation.iam.endpoint = "https://info.services.auth.zalando.com/oauth2/toke
 # Maximum number of failures before opening the circuit
 authorisation.iam.cb.maxFailures = 4
 
-# Duration of time in milliseconds after which to consider a call a failure
+# Duration in milliseconds after which to consider a call a failure
 authorisation.iam.cb.callTimeout = 2000
 
-# Duration of time in milliseconds after which to attempt to close the circuit
+# Duration in milliseconds after which to attempt to close the circuit
 authorisation.iam.cb.resetTimeout = 60000
 
+# Maximum number of retries
+authorisation.iam.maxRetries = 3
+
+# Duration in milliseconds of the exponential backoff
+authorisation.iam.retry.backoff.duration = 100
+
 # IAMClient depends on Play internal WS client so it also has to be configured.
 # The maximum time to wait when connecting to the remote host.
 # Play's default is 120 seconds

build.sbt

@@ -5,7 +5,7 @@ import scalariform.formatter.preferences._
 
 val commonSettings = Seq(
   organization := "org.zalando",
-  version := "0.2.1.1",
+  version := "0.2.2",
   scalaVersion := "2.11.8",
   scalacOptions := Seq("-unchecked", "-deprecation", "-encoding", "utf8"),
   publishTo := {
@@ -34,19 +34,23 @@ lazy val testDependencies =
 
 lazy val playDependencies =
   Seq(
-    "com.typesafe.play" %% "play-json" % playFrameworkVersion,
-    "com.typesafe.play" %% "play-ws" % playFrameworkVersion,
-    "com.typesafe.play" %% "play" % playFrameworkVersion,
-    "com.typesafe.play" %% "play-test" % playFrameworkVersion % "test",
-    "com.typesafe.play" %% "play-specs2" % playFrameworkVersion % "test"
+    "com.typesafe.play" %% "play-json"    % playFrameworkVersion,
+    "com.typesafe.play" %% "play-ws"      % playFrameworkVersion,
+    "com.typesafe.play" %% "play"         % playFrameworkVersion,
+    "com.typesafe.play" %% "play-test"    % playFrameworkVersion % "test",
+    "com.typesafe.play" %% "play-specs2"  % playFrameworkVersion % "test"
   )
 
+lazy val libraries =
+  Seq(
+    "io.zman" %% "atmos" % "2.1"
+  )
 
 lazy val root = (project in file("."))
   .settings(commonSettings: _*)
   .settings(name := "play-Zhewbacca")
-  .settings(version := "0.2.1.1")
-  .settings(libraryDependencies ++= (testDependencies ++ playDependencies))
+  .settings(version := "0.2.2")
+  .settings(libraryDependencies ++= (testDependencies ++ playDependencies ++ libraries))
   .settings(parallelExecution in Test := false)
 
 // Define a special task which does not fail when any publish task fails for any module,

src/main/scala/org/zalando/zhewbacca/IAMClient.scala

@@ -14,6 +14,9 @@ import scala.concurrent.duration._
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.control.NonFatal
 
+import atmos.dsl._
+import atmos.dsl.Slf4jSupport._
+
 /**
   * Retrieves TokenInfo for given OAuth2 token using IAM API.
   *
@@ -57,6 +60,14 @@ class IAMClient @Inject() (
     throw new IllegalArgumentException("Authorisation: Circuit Breaker reset timeout is not configured")
   ).millis
 
+  val breakerMaxRetries = config.getInt("authorisation.iam.maxRetries").getOrElse(
+    throw new IllegalArgumentException("Authorisation: Circuit Breaker max retries is not configured")
+  ).attempts
+
+  val breakerRetryBackoff = config.getInt("authorisation.iam.retry.backoff.duration").getOrElse(
+    throw new IllegalArgumentException("Authorisation: Circuit Breaker the duration of exponential backoff is not configured")
+  ).millis
+
   lazy val breaker: CircuitBreaker = new CircuitBreaker(
     actorSystem.scheduler,
     breakerMaxFailures,
@@ -72,7 +83,11 @@ class IAMClient @Inject() (
 
   override def apply(token: OAuth2Token): Future[Option[TokenInfo]] = {
     breaker.withCircuitBreaker(
-      plugableMetrics.timing(ws.url(authEndpoint).withQueryString(("access_token", token.value)).get())
+      plugableMetrics.timing(
+        retryAsync(s"Calling $authEndpoint") {
+          ws.url(authEndpoint).withQueryString(("access_token", token.value)).get()
+        }
+      )
     ).map { response =>
         response.status match {
           case OK => Some(response.json.as[TokenInfo])
@@ -85,4 +100,10 @@ class IAMClient @Inject() (
       }
   }
 
+  implicit val retryRecover = retryFor { breakerMaxRetries } using {
+    exponentialBackoff { breakerRetryBackoff }
+  } monitorWith {
+    logger.logger onRetrying logNothing onInterrupted logWarning onAborted logError
+  }
+
 }

src/test/scala/org/zalando/zhewbacca/IAMClientSpec.scala

@@ -175,6 +175,8 @@ class IAMClientSpec extends Specification {
       "authorisation.iam.cb.maxFailures" -> 4,
       "authorisation.iam.cb.callTimeout" -> 2000,
       "authorisation.iam.cb.resetTimeout" -> 60000,
+      "authorisation.iam.maxRetries" -> 3,
+      "authorisation.iam.retry.backoff.duration" -> 100,
       "play.ws.timeout.connection" -> 2000,
       "play.ws.timeout.idle" -> 2000,
       "play.ws.timeout.request" -> 2000