Merge branch 'main' into update/mockito-3-4-3.2.10.0

.github/workflows/ci.yml

@@ -28,8 +28,9 @@ jobs:
         run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt
 
       - name: Setup
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v2.3.1
         with:
+          distribution: adopt
           java-version: 11
           java-package: jdk
           architecture: x64
@@ -42,8 +43,6 @@ jobs:
 
       - name: Test
         run: sbt clean test -Dconfig.file=hat/conf/application.test.conf
-        env:
-          AWS_REGION: eu-west-1
 
       # - name: Coverage - off until we re-add Coverage
       #   run: sbt "project hat" coverage test -Dconfig.file=hat/conf/application.test.conf

.github/workflows/deploy-dev.yml

@@ -33,7 +33,7 @@ jobs:
         run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt
 
       - name: Setup Java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v2.3.1
         with:
           distribution: adopt
           java-version: 11

.github/workflows/deploy-main.yml

@@ -32,7 +32,7 @@ jobs:
         run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt
 
       - name: Setup Java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v2.3.1
         with:
           distribution: adopt
           java-version: 11

.github/workflows/deploy-release.yml

@@ -32,7 +32,7 @@ jobs:
         run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt
 
       - name: Setup Java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v2.3.1
         with:
           distribution: adopt
           java-version: 11

.github/workflows/scala-steward.yml

@@ -0,0 +1,17 @@
+name: Scala Steward
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 4 * * MON"
+
+jobs:
+  scala-steward:
+    runs-on: ubuntu-latest
+    name: Launch Scala Steward
+    steps:
+      - name: Launch Scala Steward
+        uses: scala-steward-org/scala-steward-action@v2
+        with:
+          github-token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          author-email: ${{ secrets.BOT_GITHUB_EMAIL }}

Dockerfile

@@ -1 +1 @@
-FROM ghcr.io/dataswift/base:v0.2.0
+FROM ghcr.io/dataswift/base:v0.3.0

build.sbt

@@ -48,6 +48,10 @@ lazy val hat = project
           LocalThirdParty.ScalaTestplusMockito % Test
         ),
     Test / parallelExecution := false,
+    Test / fork := true,
+    Test / envVars := Map("AWS_DEFAULT_REGION" -> "eu-west-1", "AWS_REGION" -> "eu-west-1"),
+    IntegrationTest / fork := true,
+    IntegrationTest / envVars := Map("AWS_DEFAULT_REGION" -> "eu-west-1", "AWS_REGION" -> "eu-west-1"),
     Assets / pipelineStages := Seq(digest),
     Assets / sourceDirectory := baseDirectory.value / "app" / "org" / "hatdex" / "hat" / "phata" / "assets",
     update / aggregate := false,

hat/app/org/hatdex/hat/api/controllers/Applications.scala

@@ -138,8 +138,10 @@ class Applications @Inject() (
           Owner()
         )
     ).async { implicit request =>
+      logger.info(s"Setting up application with id: $id")
       applicationsService.applicationStatus(id).flatMap { maybeStatus =>
         maybeStatus map { status =>
+          logger.info(s"Application status is: $status")
           applicationsService
             .setup(status)
             .map(s => Ok(Json.toJson(s)))

hat/app/org/hatdex/hat/api/controllers/Authentication.scala

@@ -46,7 +46,7 @@ import play.api.i18n.Lang
 import play.api.libs.json.Json
 import play.api.libs.ws.WSClient
 import play.api.mvc.{ Action, _ }
-import play.api.{ Configuration, Logger }
+import play.api.{ Configuration, Logging }
 
 import java.net.{ URLDecoder, URLEncoder }
 import javax.inject.Inject
@@ -71,12 +71,11 @@ class Authentication @Inject() (
     tokenService: MailTokenService[MailTokenUser],
     wsClient: WSClient,
     limiter: UserLimiter)
-    extends HatApiController(components, silhouette) {
+    extends HatApiController(components, silhouette)
+    with Logging {
 
   import HatJsonFormats._
 
-  private val logger = Logger(this.getClass)
-
   private val indefiniteSuccessCaching: CachedBuilder = cached
     .status(req => s"${req.host}${req.path}", 200)
     .includeStatus(404, 600)
@@ -324,7 +323,7 @@ class Authentication @Inject() (
               val token = MailTokenUser(email, isSignup = false)
               // Store that token
               tokenService.create(token).map { _ =>
-                mailer.passwordReset(email, passwordResetLink(request.host, token.id))
+                mailer.passwordReset(email, passwordResetLink(token.id, request.host))
                 response
               }
             // The user was not found, but return the "If we found an email address, we'll send the link."
@@ -401,13 +400,15 @@ class Authentication @Inject() (
       val email           = request.dynamicEnvironment.ownerEmail
       val response        = Ok(Json.toJson(SuccessResponse("You will shortly receive an email with claim instructions")))
 
+      logger.info("Handling verification request")
       // (email, applicationId) in the body
       // Look up the application (Is this in the HAT itself?  Not DEX)
       if (claimHatRequest.email == email)
         userService.listUsers
           .map(_.find(u => (u.roles.contains(Owner()) && !(u.roles.contains(Verified("email"))))))
           .flatMap {
             case Some(user) =>
+              logger.info("User found")
               val eventualClaimContext = for {
                 maybeApplication <- applicationsService
                                       .applicationStatus()(request.dynamicEnvironment, user, request)
@@ -558,35 +559,40 @@ class Authentication @Inject() (
   private def ensureValidToken(
       email: String,
       isSignup: Boolean
-    )(implicit hatServer: HatServer): Future[MailTokenUser] =
+    )(implicit hatServer: HatServer): Future[MailTokenUser] = {
+    logger.info("Checking for valid tokens")
     tokenService.retrieve(email, isSignup).flatMap {
       case Some(token) if token.isExpired =>
-        // TODO: log event for audit purpose
+        logger.info(s"Token $token is expired. Consuming the token and creating a new one")
         for {
           _ <- tokenService.consume(token.id)
           newToken <- tokenService.create(MailTokenUser(email, isSignup = isSignup))
         } yield newToken.get // TODO: Using .get is not great here
       // the underlying implementation generates non-option type and then wraps it in Option, interface not great, suggestions appreciated
       case Some(token) =>
+        logger.debug("Token found")
         Future.successful(token)
       case None =>
-        // TODO: log event for audit purpose
+        logger.info("Creating new token")
         tokenService.create(MailTokenUser(email, isSignup = isSignup)).map(_.get)
     }
+  }
 
   /**
     * Generate email verification string
     */
   private def emailVerificationLink(
-      host: String,
-      token: String,
-      verificationOptions: EmailVerificationOptions): String =
+                                     host: String,
+                                     token: String,
+                                     verificationOptions: EmailVerificationOptions): String = {
+    logger.info("Creating email verification link")
     s"$emailScheme$host/auth/verify-email/$token?${verificationOptions.asQueryParameters}"
+  }
 
   // TODO: add reset options support
   private def passwordResetLink(
-      host: String,
-      token: String): String =
+                                 host: String,
+                                 token: String): String =
     s"$emailScheme$host/auth/change-password/$token"
 
   // private def roleMatcher(rolesToMatch: Seq[UserRole], rolesRequired: Seq[UserRole]): Boolean = {

hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala

@@ -79,6 +79,12 @@ class SystemStatus @Inject() (
       }
     }
 
+  def healthReport(): Action[AnyContent] = {
+    Action.async {
+      Future.successful(Ok(Json.toJson(Map[String, String]("Status" -> "OK"))))
+    }
+  }
+
   def status(): Action[AnyContent] =
     SecuredAction(
       WithRole(Owner(), Platform()) || ContainsApplicationRole(

hat/app/org/hatdex/hat/api/service/StatsReporter.scala

@@ -28,7 +28,7 @@ import akka.actor.{ ActorSystem, Scheduler }
 import com.mohiva.play.silhouette.api.services.AuthenticatorService
 import com.mohiva.play.silhouette.impl.authenticators.JWTRS256Authenticator
 import io.dataswift.models.hat.{ DataStats, Platform }
-import org.hatdex.dex.apiV2.DexClient
+import org.hatdex.dex.apiV3.DexClient
 import org.hatdex.hat.authentication.models.HatUser
 import org.hatdex.hat.dal.ModelTranslation
 import org.hatdex.hat.dal.Tables._
@@ -68,12 +68,8 @@ class StatsReporter @Inject() (
   )
   private val statsBatchSize = 100
 
-  private val dexClient = new DexClient(
-    wsClient,
-    configuration.underlying.getString("exchange.address"),
-    configuration.underlying.getString("exchange.scheme"),
-    "v1.1"
-  )
+  private val dexAdress = configuration.underlying.getString("exchange.address")
+  private val dexClient = new DexClient(wsClient, dexAdress)
   //  val defaultSsslConfig = AkkaSSLConfig()
 
   def reportStatistics(
@@ -168,11 +164,8 @@ class StatsReporter @Inject() (
     userService.getUserByRole(Platform())(server).map(_.head)
 
   private def validateToken()(implicit server: HatServer): Future[String] = {
-    //private def applicationToken()(implicit server: HatServer): Future[String] = {
-    val resource = configuration.underlying.getString(
-        "exchange.scheme"
-      ) + configuration.underlying
-            .getString("exchange.address")
+    logger.info("Validating token")
+    val resource = configuration.underlying.getString("exchange.address")
     val customClaims = Map(
       "resource" -> Json.toJson(resource),
       "accessScope" -> Json.toJson("validate")

hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala

@@ -82,6 +82,7 @@ class ApplicationsService @Inject() (
     )(implicit hat: HatServer,
       user: HatUser,
       requestHeader: RequestHeader): Future[Option[HatApplication]] = {
+    logger.info(s"Fetching application status for app with id: $id")
     val eventuallyCleanedCache =
       if (bustCache)
         Future.sequence(
@@ -100,15 +101,28 @@ class ApplicationsService @Inject() (
                          case None          =>
                            // if any item has expired, the aggregated statuses must be refreshed
                            cache.remove(s"apps:${hat.domain}")
+                           logger.info("App was not found in the cache. Making request")
                            for {
-                             maybeApp <- trustedApplicationProvider.application(id)
-                             setup <- applicationSetupStatus(id)(hat.db)
-                             status <- FutureTransformations.transform(
-                                         maybeApp.map(refetchApplicationsStatus(_, Seq(setup).flatten))
-                                       )
-                             _ <- status
-                                    .map(s => cache.set(appCacheKey(id), s._1, applicationsCacheDuration))
-                                    .getOrElse(Future.successful(Done))
+                             maybeApp <- {
+                               logger.debug("Fetching app")
+                               trustedApplicationProvider.application(id)
+                             }
+                             setup <- {
+                               logger.debug("Fetching app status")
+                               applicationSetupStatus(id)(hat.db)
+                             }
+                             status <- {
+                               logger.debug("Making the transformation by refetching app")
+                               FutureTransformations.transform(
+                                 maybeApp.map(refetchApplicationsStatus(_, Seq(setup).flatten))
+                               )
+                             }
+                             _ <- {
+                               logger.debug("Storing the app in the cache")
+                               status
+                                 .map(s => cache.set(appCacheKey(id), s._1, applicationsCacheDuration))
+                                 .getOrElse(Future.successful(Done))
+                             }
                            } yield status.map(_._1)
                        }
     } yield application
@@ -117,22 +131,39 @@ class ApplicationsService @Inject() (
   def applicationStatus(
     )(implicit hat: HatServer,
       user: HatUser,
-      requestHeader: RequestHeader): Future[Seq[HatApplication]] =
+      requestHeader: RequestHeader): Future[Seq[HatApplication]] = {
+    logger.info(s"Fetching application status")
     cache
       .get[Seq[HatApplication]](s"apps:${hat.domain}")
       .flatMap {
-        case Some(applications) => Future.successful(applications)
+        case Some(applications) =>
+          logger.debug("Application was found in the cache")
+          Future.successful(applications)
         case None =>
+          logger.debug("Application was not found in the cache. Fetching application")
           for {
-            apps <- trustedApplicationProvider.applications // potentially caching
-            setup <- applicationSetupStatus()(hat.db) // database
-            statuses <- Future.sequence(apps.map(refetchApplicationsStatus(_, setup)))
+            apps <- {
+              logger.debug("Fetching applications")
+              trustedApplicationProvider.applications // potentially caching
+            }
+            setup <- {
+              logger.debug("Fetching application setup status")
+              applicationSetupStatus()(hat.db) // database
+            }
+            statuses <- {
+              logger.debug(s"Refreshing application status")
+              Future.sequence(apps.map(refetchApplicationsStatus(_, setup)))
+            }
             apps = statuses.map(_._1)
-            _ <- if (statuses.forall(_._2))
-                   cache.set(s"apps:${hat.domain}", apps, applicationsCacheDuration)
-                 else Future.successful(Done)
+            _ <- {
+              logger.debug("Attempting to save applications in cache")
+              if (statuses.forall(_._2))
+                cache.set(s"apps:${hat.domain}", apps, applicationsCacheDuration)
+              else Future.successful(Done)
+            }
           } yield apps
       }
+  }
 
   private def refetchApplicationsStatus(
       app: Application,
@@ -291,6 +322,7 @@ class ApplicationsService @Inject() (
     )(implicit hat: HatServer,
       user: HatUser,
       requestHeader: RequestHeader): Future[HatApplication] = {
+    logger.info(s"Setting app application: ${application.application.id}")
     val appSetup = for {
       // Create and enable the data debit
       _ <- enableAssociatedDataDebit(application)

hat/app/org/hatdex/hat/api/service/applications/TrustedApplicationProviderDex.scala

@@ -1,8 +1,8 @@
 package org.hatdex.hat.api.service.applications
 
 import io.dataswift.models.hat.applications.Application
-import org.hatdex.dex.apiV2.DexClient
-import org.hatdex.dex.apiV2.Errors.ApiException
+import org.hatdex.dex.apiV3.DexClient
+import org.hatdex.dex.apiV3.Errors.ApiException
 import org.hatdex.hat.api.service.RemoteExecutionContext
 import play.api.cache.AsyncCacheApi
 import play.api.libs.ws.WSClient
@@ -21,12 +21,8 @@ class TrustedApplicationProviderDex @Inject() (
 
   private val logger = Logger(this.getClass)
 
-  private val dexClient = new DexClient(
-    wsClient,
-    configuration.underlying.getString("exchange.address"),
-    configuration.underlying.getString("exchange.scheme"),
-    "v1.1"
-  )
+  private val dexAddress = configuration.underlying.getString("exchange.address")
+  private val dexClient  = new DexClient(wsClient, dexAddress)
 
   private val applicationsCacheDuration = configuration.get[FiniteDuration]("application-cache-ttl")
 
@@ -38,7 +34,7 @@ class TrustedApplicationProviderDex @Inject() (
       "apps:dexApplications",
       applicationsCacheDuration
     ) {
-      dexClient.applications(includeUnpublished = includeUnpublished)
+      dexClient.applications(Some(includeUnpublished))
     }
 
   def application(id: String): Future[Option[Application]] =