diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 921f9ae07..5b998df74 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -36,8 +36,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYER_STAGING_SECRET_ACCESS_KEY }} run: | cd ./hat/app/org/hatdex/hat/phata/assets/ - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1 + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1 cp rumpel/*.js* js/ cp rumpel/*.css stylesheets/ gzip --keep --force js/*.bundle.js diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml index ae32e0ab0..775cbcb53 100644 --- a/.github/workflows/deploy-main.yml +++ b/.github/workflows/deploy-main.yml @@ -35,8 +35,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DEPLOYER_SANDBOX_SECRET_ACCESS_KEY }} run: | cd ./hat/app/org/hatdex/hat/phata/assets/ - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1 + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1 cp rumpel/*.js* js/ cp rumpel/*.css stylesheets/ gzip --keep --force js/*.bundle.js diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml index ff014eba6..951133ecd 100644 --- a/.github/workflows/deploy-release.yml +++ b/.github/workflows/deploy-release.yml @@ -35,8 +35,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} run: | cd ./hat/app/org/hatdex/hat/phata/assets/ - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive - aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL }} rumpel --recursive --region eu-west-1 + aws s3 cp s3://${{ env.RUMPEL_BUCKET }}/${{ env.RUMPEL_ALT }} alt-rumpel --recursive --region eu-west-1 cp rumpel/*.js* js/ cp rumpel/*.css stylesheets/ gzip --keep --force js/*.bundle.js diff --git a/build.sbt b/build.sbt index 0bada9748..a946f6fc9 100644 --- a/build.sbt +++ b/build.sbt @@ -47,7 +47,8 @@ lazy val hat = project Library.scalaGuice, Library.circeConfig, Library.ContractLibrary.adjudicator, - Library.Utils.apacheCommonLang + Library.Utils.apacheCommonLang, + Library.Prometheus.filters ), libraryDependencies := (buildEnv.value match { case BuildEnv.Developement | BuildEnv.Test => diff --git a/hat/app/org/hatdex/hat/filters/PrometheusFilters.scala b/hat/app/org/hatdex/hat/filters/PrometheusFilters.scala new file mode 100644 index 000000000..9d2ac2a17 --- /dev/null +++ b/hat/app/org/hatdex/hat/filters/PrometheusFilters.scala @@ -0,0 +1,10 @@ +package org.hatdex.hat.filters + +import javax.inject.Inject +import com.github.stijndehaes.playprometheusfilters.filters.{ StatusAndRouteLatencyFilter, StatusCounterFilter } +import play.api.http.DefaultHttpFilters + +class PrometheusFilters @Inject() ( + statusCounterFilter: StatusCounterFilter, + statusAndRouteLatencyFilter: StatusAndRouteLatencyFilter) + extends DefaultHttpFilters(statusCounterFilter, statusAndRouteLatencyFilter) diff --git a/hat/conf/application.conf b/hat/conf/application.conf index ed54abd97..5f3b45ef4 100644 --- a/hat/conf/application.conf +++ b/hat/conf/application.conf @@ -163,3 +163,4 @@ include "she.conf" include "phata.conf" include "regions.conf" + diff --git a/hat/conf/play.conf b/hat/conf/play.conf index 8112833c8..1e1ae57af 100644 --- a/hat/conf/play.conf +++ b/hat/conf/play.conf @@ -8,6 +8,7 @@ play { secret.key = ${?APPLICATION_SECRET} errorHandler = "org.hatdex.hat.utils.ErrorHandler" forwarded.trustedProxies=["0.0.0.0/0", "::/0"] + filters = "org.hatdex.hat.filters.PrometheusFilters" } i18n.langs = ["en", "pt", "pl"] diff --git a/hat/conf/v26.routes b/hat/conf/v26.routes index 12959a522..53ffe7385 100644 --- a/hat/conf/v26.routes +++ b/hat/conf/v26.routes @@ -81,4 +81,8 @@ GET /she/static/$endpoint<[0-9a-z-/]+> # Contract Data POST /contract-data/read/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.ContractData.readContractData(namespace, endpoint, orderBy: Option[String], ordering: Option[String], skip: Option[Int], take: Option[Int]) POST /contract-data/create/$namespace<[0-9a-z-]+>/$endpoint<[0-9a-z-/]+> org.hatdex.hat.api.controllers.ContractData.createContractData(namespace, endpoint, skipErrors: Option[Boolean]) -PUT /contract-data/update/$namespace<[0-9a-z-]+> org.hatdex.hat.api.controllers.ContractData.updateContractData(namespace) \ No newline at end of file +PUT /contract-data/update/$namespace<[0-9a-z-]+> org.hatdex.hat.api.controllers.ContractData.updateContractData(namespace) + +# Metrics +# Hide for now +# GET /metrics com.github.stijndehaes.playprometheusfilters.controllers.PrometheusController.getMetrics diff --git a/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala b/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala index 47edab1f0..3a11662c9 100644 --- a/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala +++ b/hat/test/org/hatdex/hat/api/service/applications/ApplicationsServiceSpec.scala @@ -186,10 +186,11 @@ class ApplicationsServiceSpec(implicit ee: ExecutionEnv) val service = application.injector.instanceOf[ApplicationsService] val result = for { app <- service.applicationStatus(notablesAppExternalFailing.id) - _ <- service.setup(app.get) + appSetupResponse <- service.setup(app.get) setup <- service.applicationStatus(notablesAppExternalFailing.id) } yield { setup must beSome + appSetupResponse.setup must beTrue setup.get.setup must beTrue setup.get.active must beFalse } diff --git a/hat/test/org/hatdex/hat/filters/PrometheusFiltersSpec.scala b/hat/test/org/hatdex/hat/filters/PrometheusFiltersSpec.scala new file mode 100644 index 000000000..4346f2e27 --- /dev/null +++ b/hat/test/org/hatdex/hat/filters/PrometheusFiltersSpec.scala @@ -0,0 +1,129 @@ +// This is set as it is to access the private members of the filters to test them. +package com.github.stijndehaes.playprometheusfilters.filters + +/* Test cases based on the archived project: https://github.com/stijndehaes/play-prometheus-filters */ + +import org.specs2.mock.Mockito +import play.api.test.{ FakeRequest, PlaySpecification } +import akka.actor.ActorSystem +import akka.stream.ActorMaterializer +import play.api.mvc._ +import play.api.test.Helpers._ +import play.api.test._ +import play.api.routing.Router +import io.prometheus.client.CollectorRegistry +import play.api.libs.typedmap.TypedMap +import play.api.routing.HandlerDef +import play.api.mvc.{ AbstractController, ControllerComponents } +import javax.inject.Inject +import scala.collection.JavaConverters._ + +class PrometheusFiltersSpec extends PlaySpecification with Mockito { + implicit val system = ActorSystem() + implicit val executor = system.dispatcher + implicit val materializer = ActorMaterializer() + + sequential + + "LatencyFilter" should { + "Measure the latency" in { + val promFilter: LatencyFilter = new LatencyFilter(mock[CollectorRegistry]) + val fakeRequest = FakeRequest() + val action = new MockController(stubControllerComponents()).ok + + await(promFilter(action)(fakeRequest).run()) + + val metrics = promFilter.requestLatency.collect() + metrics must have size 1 + val samples = metrics.get(0).samples + + val countSample = samples.get(samples.size() - 2) + countSample.value mustEqual 1.0 + countSample.labelValues must have size 0 + } + } + + "StatusAndRouteLatencyFilter" should { + "Measure the latency and status" in { + val expectedLabelCount: Long = 5 + val expectedMethod = "test" + val expectedStatus: String = "200" + val expectedControllerName = "promController" + val expectedPath = "/path" + val expectedVerb = "GET" + val listOfMatches = java.util.Arrays.asList(expectedMethod, expectedStatus, expectedControllerName, expectedPath, expectedVerb) + + + val promFilter = new StatusAndRouteLatencyFilter(mock[CollectorRegistry]) + val fakeRequest = FakeRequest().withAttrs( + TypedMap( + Router.Attrs.HandlerDef -> HandlerDef(null, + null, + expectedControllerName, + expectedMethod, + null, + expectedVerb, + expectedPath, + null, + null + ) + ) + ) + + val action = + new MockController(stubControllerComponents()).ok + + await(promFilter(action)(fakeRequest).run()) + + val metrics = promFilter.requestLatency.collect() + metrics must have size 1 + val samples = metrics.get(0).samples + val countSample = samples.get(samples.size() - 2) + + countSample.value mustEqual 1.0 + countSample.labelValues.size mustEqual expectedLabelCount + countSample.labelValues mustEqual listOfMatches + } + } + + "StatusCounterFilter" should { + "Count the requests with status" in { + val promFilter = new StatusCounterFilter(mock[CollectorRegistry]) + val fakeRequest = FakeRequest() + val actionOK = new MockController(stubControllerComponents()).ok + val actionError = new MockController(stubControllerComponents()).error + + val bothRequests = for { + ok <- promFilter(actionOK)(fakeRequest).run() + ko <- promFilter(actionError)(fakeRequest).run() + } yield (ok, ko) + + await(bothRequests) + + val metrics = promFilter.requestCounter.collect() + metrics.size mustEqual 1 + val samplesOk = metrics.get(0).samples.get(0) + val samplesError = metrics.get(0).samples.get(1) + + samplesOk.value mustEqual 1.0 + samplesOk.labelValues.size mustEqual 1 + samplesOk.labelValues.get(0) mustEqual "200" + + samplesError.value mustEqual 1.0 + samplesError.labelValues.size mustEqual 1 + samplesError.labelValues.get(0) mustEqual "404" + } + } +} + +class MockController @Inject() (cc: ControllerComponents) extends AbstractController(cc) { + def ok = + Action { + Ok("ok") + } + + def error = + Action { + NotFound("error") + } +} diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 311692a3a..88a743f50 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -109,5 +109,9 @@ object Dependencies { val adjudicator = "io.dataswift" %% "adjudicatorlib" % Versions.adjudicator } + object Prometheus { + val filters = "com.github.stijndehaes" %% "play-prometheus-filters" % "0.4.0" + } + } } diff --git a/project/build.properties b/project/build.properties index 7e14a9e86..ebb77ddb6 100644 --- a/project/build.properties +++ b/project/build.properties @@ -22,4 +22,4 @@ # 2 / 2017 # -sbt.version=1.3.8 +sbt.version=1.3.13