Dev (#302)

* DAS 92 add put (#280) Co-authored-by: Augustinas Markevicius <[email protected]> Co-authored-by: Piotr Perzyna <[email protected]> * DAS 92 add put 2 (#283) Co-authored-by: Augustinas Markevicius <[email protected]> Co-authored-by: Piotr Perzyna <[email protected]> * Improve deployment (#284) * update github action * cleanup * DAS 92 add put 2 (#286) Co-authored-by: Augustinas Markevicius <[email protected]> Co-authored-by: Piotr Perzyna <[email protected]> * Update README.md (#287) * DAS 98 - Add Prometheus metrics. (#294) * add region to aws cli (#296) * Updated to sbt 1.3.13 (#301) * Updated to sbt 1.3.13 * Fixed a test * fix one test * Update v26.routes Hide the /metrics route for now. Co-authored-by: Ty <[email protected]> Co-authored-by: Augustinas Markevicius <[email protected]> Co-authored-by: Piotr Perzyna <[email protected]>
dataswift · Feb 11, 2021 · ad379e0 · ad379e0
1 parent 749efe9
commit ad379e0
Show file tree

Hide file tree

Showing 12 changed files with 161 additions and 10 deletions.
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
@@ -36,8 +36,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYER_STAGING_SECRET_ACCESS_KEY }}
         run: |
           cd ./hat/app/org/hatdex/hat/phata/assets/
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1
           cp rumpel/*.js* js/
           cp rumpel/*.css stylesheets/
           gzip --keep --force js/*.bundle.js

diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml
@@ -35,8 +35,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYER_SANDBOX_SECRET_ACCESS_KEY }}
         run: |
           cd ./hat/app/org/hatdex/hat/phata/assets/
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1
           cp rumpel/*.js* js/
           cp rumpel/*.css stylesheets/
           gzip --keep --force js/*.bundle.js

diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml
@@ -35,8 +35,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }}
         run: |
           cd ./hat/app/org/hatdex/hat/phata/assets/
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive
-          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1
+          aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1
           cp rumpel/*.js* js/
           cp rumpel/*.css stylesheets/
           gzip --keep --force js/*.bundle.js

diff --git a/build.sbt b/build.sbt
@@ -47,7 +47,8 @@ lazy val hat = project
           Library.scalaGuice,
           Library.circeConfig,
           Library.ContractLibrary.adjudicator,
-          Library.Utils.apacheCommonLang
+          Library.Utils.apacheCommonLang,
+          Library.Prometheus.filters
         ),
     libraryDependencies := (buildEnv.value match {
           case BuildEnv.Developement | BuildEnv.Test =>

diff --git a/hat/app/org/hatdex/hat/filters/PrometheusFilters.scala b/hat/app/org/hatdex/hat/filters/PrometheusFilters.scala
@@ -0,0 +1,10 @@
+package org.hatdex.hat.filters
+
+import javax.inject.Inject
+import com.github.stijndehaes.playprometheusfilters.filters.{ StatusAndRouteLatencyFilter, StatusCounterFilter }
+import play.api.http.DefaultHttpFilters
+
+class PrometheusFilters @Inject() (
+    statusCounterFilter: StatusCounterFilter,
+    statusAndRouteLatencyFilter: StatusAndRouteLatencyFilter)
+    extends DefaultHttpFilters(statusCounterFilter, statusAndRouteLatencyFilter)
diff --git a/hat/conf/application.conf b/hat/conf/application.conf
@@ -163,3 +163,4 @@ include "she.conf"
 include "phata.conf"
 
 include "regions.conf"
+
diff --git a/hat/conf/play.conf b/hat/conf/play.conf
@@ -8,6 +8,7 @@ play {
     secret.key = ${?APPLICATION_SECRET}
     errorHandler = "org.hatdex.hat.utils.ErrorHandler"
     forwarded.trustedProxies=["0.0.0.0/0", "::/0"]
+    filters = "org.hatdex.hat.filters.PrometheusFilters"
   }
 
   i18n.langs = ["en", "pt", "pl"]

diff --git a/hat/conf/v26.routes b/hat/conf/v26.routes
@@ -81,4 +81,8 @@ GET           /she/static/$endpoint<[0-9a-z-/]+>
 # Contract Data
 POST          /contract-data/read/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+>              org.hatdex.hat.api.controllers.ContractData.readContractData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int])
 POST          /contract-data/create/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+>            org.hatdex.hat.api.controllers.ContractData.createContractData(namespace, endpoint, skipErrors: Option[Boolean])
-PUT           /contract-data/update/$namespace<[0-9a-z-]+>                                   org.hatdex.hat.api.controllers.ContractData.updateContractData(namespace)
+PUT           /contract-data/update/$namespace<[0-9a-z-]+>                                   org.hatdex.hat.api.controllers.ContractData.updateContractData(namespace)
+
+# Metrics
+# Hide for now
+# GET           /metrics                                                                       com.github.stijndehaes.playprometheusfilters.controllers.PrometheusController.getMetrics
diff --git a/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala b/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala
@@ -186,10 +186,11 @@ class ApplicationsServiceSpec(implicit ee: ExecutionEnv)
       val service = application.injector.instanceOf[ApplicationsService]
       val result = for {
         app <- service.applicationStatus(notablesAppExternalFailing.id)
-        _ <- service.setup(app.get)
+        appSetupResponse <- service.setup(app.get)
         setup <- service.applicationStatus(notablesAppExternalFailing.id)
       } yield {
         setup must beSome
+        appSetupResponse.setup must beTrue
         setup.get.setup must beTrue
         setup.get.active must beFalse
       }

diff --git a/hat/test/org/hatdex/hat/filters/PrometheusFiltersSpec.scala b/hat/test/org/hatdex/hat/filters/PrometheusFiltersSpec.scala
@@ -0,0 +1,129 @@
+// This is set as it is to access the private members of the filters to test them.
+package com.github.stijndehaes.playprometheusfilters.filters
+
+/* Test cases based on the archived project: https://github.com/stijndehaes/play-prometheus-filters */
+
+import org.specs2.mock.Mockito
+import play.api.test.{ FakeRequest, PlaySpecification }
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import play.api.mvc._
+import play.api.test.Helpers._
+import play.api.test._
+import play.api.routing.Router
+import io.prometheus.client.CollectorRegistry
+import play.api.libs.typedmap.TypedMap
+import play.api.routing.HandlerDef
+import play.api.mvc.{ AbstractController, ControllerComponents }
+import javax.inject.Inject
+import scala.collection.JavaConverters._
+
+class PrometheusFiltersSpec extends PlaySpecification with Mockito {
+  implicit val system       = ActorSystem()
+  implicit val executor     = system.dispatcher
+  implicit val materializer = ActorMaterializer()
+
+  sequential
+
+  "LatencyFilter" should {
+    "Measure the latency" in {
+      val promFilter: LatencyFilter = new LatencyFilter(mock[CollectorRegistry])
+      val fakeRequest               = FakeRequest()
+      val action                    = new MockController(stubControllerComponents()).ok
+
+      await(promFilter(action)(fakeRequest).run())
+
+      val metrics = promFilter.requestLatency.collect()
+      metrics must have size 1
+      val samples = metrics.get(0).samples
+
+      val countSample = samples.get(samples.size() - 2)
+      countSample.value mustEqual 1.0
+      countSample.labelValues must have size 0
+    }
+  }
+
+  "StatusAndRouteLatencyFilter" should {
+    "Measure the latency and status" in {
+      val expectedLabelCount: Long = 5
+      val expectedMethod           = "test"
+      val expectedStatus: String   = "200"
+      val expectedControllerName   = "promController"
+      val expectedPath             = "/path"
+      val expectedVerb             = "GET"
+      val listOfMatches            = java.util.Arrays.asList(expectedMethod, expectedStatus, expectedControllerName, expectedPath, expectedVerb)
+
+
+      val promFilter = new StatusAndRouteLatencyFilter(mock[CollectorRegistry])
+      val fakeRequest = FakeRequest().withAttrs(
+        TypedMap(
+          Router.Attrs.HandlerDef -> HandlerDef(null,
+                                                null,
+                                                expectedControllerName,
+                                                expectedMethod,
+                                                null,
+                                                expectedVerb,
+                                                expectedPath,
+                                                null,
+                                                null
+              )
+        )
+      )
+
+      val action =
+        new MockController(stubControllerComponents()).ok
+
+      await(promFilter(action)(fakeRequest).run())
+
+      val metrics = promFilter.requestLatency.collect()
+      metrics must have size 1
+      val samples     = metrics.get(0).samples
+      val countSample = samples.get(samples.size() - 2)
+
+      countSample.value mustEqual 1.0
+      countSample.labelValues.size mustEqual expectedLabelCount
+      countSample.labelValues mustEqual listOfMatches
+    }
+  }
+
+  "StatusCounterFilter" should {
+    "Count the requests with status" in {
+      val promFilter  = new StatusCounterFilter(mock[CollectorRegistry])
+      val fakeRequest = FakeRequest()
+      val actionOK    = new MockController(stubControllerComponents()).ok
+      val actionError = new MockController(stubControllerComponents()).error
+
+      val bothRequests = for {
+        ok <- promFilter(actionOK)(fakeRequest).run()
+        ko <- promFilter(actionError)(fakeRequest).run()
+      } yield (ok, ko)
+
+      await(bothRequests)
+
+      val metrics = promFilter.requestCounter.collect()
+      metrics.size mustEqual 1
+      val samplesOk    = metrics.get(0).samples.get(0)
+      val samplesError = metrics.get(0).samples.get(1)
+
+      samplesOk.value mustEqual 1.0
+      samplesOk.labelValues.size mustEqual 1
+      samplesOk.labelValues.get(0) mustEqual "200"
+
+      samplesError.value mustEqual 1.0
+      samplesError.labelValues.size mustEqual 1
+      samplesError.labelValues.get(0) mustEqual "404"
+    }
+  }
+}
+
+class MockController @Inject() (cc: ControllerComponents) extends AbstractController(cc) {
+  def ok =
+    Action {
+      Ok("ok")
+    }
+
+  def error =
+    Action {
+      NotFound("error")
+    }
+}
diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -109,5 +109,9 @@ object Dependencies {
       val adjudicator = "io.dataswift" %% "adjudicatorlib" % Versions.adjudicator
     }
 
+    object Prometheus {
+      val filters = "com.github.stijndehaes" %% "play-prometheus-filters" % "0.4.0"
+    }
+
   }
 }
diff --git a/project/build.properties b/project/build.properties
@@ -22,4 +22,4 @@
 # 2 / 2017
 #
 
-sbt.version=1.3.8
+sbt.version=1.3.13
Original file line number	Diff line number	Diff line change
Expand Up		@@ -163,3 +163,4 @@ include "she.conf"
		include "phata.conf"

		include "regions.conf"