diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe20e8595..98da11b28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,9 @@ jobs: run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt - name: Setup - uses: actions/setup-java@v1 + uses: actions/setup-java@v2.3.1 with: + distribution: adopt java-version: 11 java-package: jdk architecture: x64 @@ -42,8 +43,6 @@ jobs: - name: Test run: sbt clean test -Dconfig.file=hat/conf/application.test.conf - env: - AWS_REGION: eu-west-1 # - name: Coverage - off until we re-add Coverage # run: sbt "project hat" coverage test -Dconfig.file=hat/conf/application.test.conf diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 82f2ccaaa..1384acefc 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -33,7 +33,7 @@ jobs: run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v2.3.1 with: distribution: adopt java-version: 11 diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml index 5177036ca..8f88ff7c4 100644 --- a/.github/workflows/deploy-main.yml +++ b/.github/workflows/deploy-main.yml @@ -32,7 +32,7 @@ jobs: run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v2.3.1 with: distribution: adopt java-version: 11 diff --git a/.github/workflows/deploy-release.yml b/.github/workflows/deploy-release.yml index 3a428939c..ea9e6c337 100644 --- a/.github/workflows/deploy-release.yml +++ b/.github/workflows/deploy-release.yml @@ -32,7 +32,7 @@ jobs: run: curl https://${{ secrets.ARTIFACTS_CACHE_BUCKET }}/resolvers.sbt --create-dirs -o ~/.sbt/resolvers.sbt - name: Setup Java - uses: actions/setup-java@v2 + uses: actions/setup-java@v2.3.1 with: distribution: adopt java-version: 11 diff --git a/.github/workflows/scala-steward.yml b/.github/workflows/scala-steward.yml new file mode 100644 index 000000000..f7b62bdee --- /dev/null +++ b/.github/workflows/scala-steward.yml @@ -0,0 +1,17 @@ +name: Scala Steward + +on: + workflow_dispatch: + schedule: + - cron: "0 4 * * MON" + +jobs: + scala-steward: + runs-on: ubuntu-latest + name: Launch Scala Steward + steps: + - name: Launch Scala Steward + uses: scala-steward-org/scala-steward-action@v2 + with: + github-token: ${{ secrets.BOT_GITHUB_TOKEN }} + author-email: ${{ secrets.BOT_GITHUB_EMAIL }} diff --git a/Dockerfile b/Dockerfile index c5777b1d9..01755c01b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -FROM ghcr.io/dataswift/base:v0.2.0 +FROM ghcr.io/dataswift/base:v0.3.0 \ No newline at end of file diff --git a/build.sbt b/build.sbt index ac1edcf7a..c56dbf064 100644 --- a/build.sbt +++ b/build.sbt @@ -48,6 +48,10 @@ lazy val hat = project LocalThirdParty.ScalaTestplusMockito % Test ), Test / parallelExecution := false, + Test / fork := true, + Test / envVars := Map("AWS_DEFAULT_REGION" -> "eu-west-1", "AWS_REGION" -> "eu-west-1"), + IntegrationTest / fork := true, + IntegrationTest / envVars := Map("AWS_DEFAULT_REGION" -> "eu-west-1", "AWS_REGION" -> "eu-west-1"), Assets / pipelineStages := Seq(digest), Assets / sourceDirectory := baseDirectory.value / "app" / "org" / "hatdex" / "hat" / "phata" / "assets", update / aggregate := false, diff --git a/hat/app/org/hatdex/hat/api/controllers/Applications.scala b/hat/app/org/hatdex/hat/api/controllers/Applications.scala index eab496ac1..7d4942d6d 100644 --- a/hat/app/org/hatdex/hat/api/controllers/Applications.scala +++ b/hat/app/org/hatdex/hat/api/controllers/Applications.scala @@ -138,8 +138,10 @@ class Applications @Inject() ( Owner() ) ).async { implicit request => + logger.info(s"Setting up application with id: $id") applicationsService.applicationStatus(id).flatMap { maybeStatus => maybeStatus map { status => + logger.info(s"Application status is: $status") applicationsService .setup(status) .map(s => Ok(Json.toJson(s))) diff --git a/hat/app/org/hatdex/hat/api/controllers/Authentication.scala b/hat/app/org/hatdex/hat/api/controllers/Authentication.scala index ff7dc96dd..20969c4a1 100644 --- a/hat/app/org/hatdex/hat/api/controllers/Authentication.scala +++ b/hat/app/org/hatdex/hat/api/controllers/Authentication.scala @@ -46,7 +46,7 @@ import play.api.i18n.Lang import play.api.libs.json.Json import play.api.libs.ws.WSClient import play.api.mvc.{ Action, _ } -import play.api.{ Configuration, Logger } +import play.api.{ Configuration, Logging } import java.net.{ URLDecoder, URLEncoder } import javax.inject.Inject @@ -71,12 +71,11 @@ class Authentication @Inject() ( tokenService: MailTokenService[MailTokenUser], wsClient: WSClient, limiter: UserLimiter) - extends HatApiController(components, silhouette) { + extends HatApiController(components, silhouette) + with Logging { import HatJsonFormats._ - private val logger = Logger(this.getClass) - private val indefiniteSuccessCaching: CachedBuilder = cached .status(req => s"${req.host}${req.path}", 200) .includeStatus(404, 600) @@ -324,7 +323,7 @@ class Authentication @Inject() ( val token = MailTokenUser(email, isSignup = false) // Store that token tokenService.create(token).map { _ => - mailer.passwordReset(email, passwordResetLink(request.host, token.id)) + mailer.passwordReset(email, passwordResetLink(token.id, request.host)) response } // The user was not found, but return the "If we found an email address, we'll send the link." @@ -401,6 +400,7 @@ class Authentication @Inject() ( val email = request.dynamicEnvironment.ownerEmail val response = Ok(Json.toJson(SuccessResponse("You will shortly receive an email with claim instructions"))) + logger.info("Handling verification request") // (email, applicationId) in the body // Look up the application (Is this in the HAT itself? Not DEX) if (claimHatRequest.email == email) @@ -408,6 +408,7 @@ class Authentication @Inject() ( .map(_.find(u => (u.roles.contains(Owner()) && !(u.roles.contains(Verified("email")))))) .flatMap { case Some(user) => + logger.info("User found") val eventualClaimContext = for { maybeApplication <- applicationsService .applicationStatus()(request.dynamicEnvironment, user, request) @@ -558,35 +559,40 @@ class Authentication @Inject() ( private def ensureValidToken( email: String, isSignup: Boolean - )(implicit hatServer: HatServer): Future[MailTokenUser] = + )(implicit hatServer: HatServer): Future[MailTokenUser] = { + logger.info("Checking for valid tokens") tokenService.retrieve(email, isSignup).flatMap { case Some(token) if token.isExpired => - // TODO: log event for audit purpose + logger.info(s"Token $token is expired. Consuming the token and creating a new one") for { _ <- tokenService.consume(token.id) newToken <- tokenService.create(MailTokenUser(email, isSignup = isSignup)) } yield newToken.get // TODO: Using .get is not great here // the underlying implementation generates non-option type and then wraps it in Option, interface not great, suggestions appreciated case Some(token) => + logger.debug("Token found") Future.successful(token) case None => - // TODO: log event for audit purpose + logger.info("Creating new token") tokenService.create(MailTokenUser(email, isSignup = isSignup)).map(_.get) } + } /** * Generate email verification string */ private def emailVerificationLink( - host: String, - token: String, - verificationOptions: EmailVerificationOptions): String = + host: String, + token: String, + verificationOptions: EmailVerificationOptions): String = { + logger.info("Creating email verification link") s"$emailScheme$host/auth/verify-email/$token?${verificationOptions.asQueryParameters}" + } // TODO: add reset options support private def passwordResetLink( - host: String, - token: String): String = + host: String, + token: String): String = s"$emailScheme$host/auth/change-password/$token" // private def roleMatcher(rolesToMatch: Seq[UserRole], rolesRequired: Seq[UserRole]): Boolean = { diff --git a/hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala b/hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala index c428e0980..574a5f1a0 100644 --- a/hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala +++ b/hat/app/org/hatdex/hat/api/controllers/SystemStatus.scala @@ -79,6 +79,12 @@ class SystemStatus @Inject() ( } } + def healthReport(): Action[AnyContent] = { + Action.async { + Future.successful(Ok(Json.toJson(Map[String, String]("Status" -> "OK")))) + } + } + def status(): Action[AnyContent] = SecuredAction( WithRole(Owner(), Platform()) || ContainsApplicationRole( diff --git a/hat/app/org/hatdex/hat/api/service/StatsReporter.scala b/hat/app/org/hatdex/hat/api/service/StatsReporter.scala index 8ac268db4..624574525 100644 --- a/hat/app/org/hatdex/hat/api/service/StatsReporter.scala +++ b/hat/app/org/hatdex/hat/api/service/StatsReporter.scala @@ -28,7 +28,7 @@ import akka.actor.{ ActorSystem, Scheduler } import com.mohiva.play.silhouette.api.services.AuthenticatorService import com.mohiva.play.silhouette.impl.authenticators.JWTRS256Authenticator import io.dataswift.models.hat.{ DataStats, Platform } -import org.hatdex.dex.apiV2.DexClient +import org.hatdex.dex.apiV3.DexClient import org.hatdex.hat.authentication.models.HatUser import org.hatdex.hat.dal.ModelTranslation import org.hatdex.hat.dal.Tables._ @@ -68,12 +68,8 @@ class StatsReporter @Inject() ( ) private val statsBatchSize = 100 - private val dexClient = new DexClient( - wsClient, - configuration.underlying.getString("exchange.address"), - configuration.underlying.getString("exchange.scheme"), - "v1.1" - ) + private val dexAdress = configuration.underlying.getString("exchange.address") + private val dexClient = new DexClient(wsClient, dexAdress) // val defaultSsslConfig = AkkaSSLConfig() def reportStatistics( @@ -168,11 +164,8 @@ class StatsReporter @Inject() ( userService.getUserByRole(Platform())(server).map(_.head) private def validateToken()(implicit server: HatServer): Future[String] = { - //private def applicationToken()(implicit server: HatServer): Future[String] = { - val resource = configuration.underlying.getString( - "exchange.scheme" - ) + configuration.underlying - .getString("exchange.address") + logger.info("Validating token") + val resource = configuration.underlying.getString("exchange.address") val customClaims = Map( "resource" -> Json.toJson(resource), "accessScope" -> Json.toJson("validate") diff --git a/hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala b/hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala index 62ec1b8fc..a7ede9ad5 100644 --- a/hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala +++ b/hat/app/org/hatdex/hat/api/service/applications/ApplicationsService.scala @@ -82,6 +82,7 @@ class ApplicationsService @Inject() ( )(implicit hat: HatServer, user: HatUser, requestHeader: RequestHeader): Future[Option[HatApplication]] = { + logger.info(s"Fetching application status for app with id: $id") val eventuallyCleanedCache = if (bustCache) Future.sequence( @@ -100,15 +101,28 @@ class ApplicationsService @Inject() ( case None => // if any item has expired, the aggregated statuses must be refreshed cache.remove(s"apps:${hat.domain}") + logger.info("App was not found in the cache. Making request") for { - maybeApp <- trustedApplicationProvider.application(id) - setup <- applicationSetupStatus(id)(hat.db) - status <- FutureTransformations.transform( - maybeApp.map(refetchApplicationsStatus(_, Seq(setup).flatten)) - ) - _ <- status - .map(s => cache.set(appCacheKey(id), s._1, applicationsCacheDuration)) - .getOrElse(Future.successful(Done)) + maybeApp <- { + logger.debug("Fetching app") + trustedApplicationProvider.application(id) + } + setup <- { + logger.debug("Fetching app status") + applicationSetupStatus(id)(hat.db) + } + status <- { + logger.debug("Making the transformation by refetching app") + FutureTransformations.transform( + maybeApp.map(refetchApplicationsStatus(_, Seq(setup).flatten)) + ) + } + _ <- { + logger.debug("Storing the app in the cache") + status + .map(s => cache.set(appCacheKey(id), s._1, applicationsCacheDuration)) + .getOrElse(Future.successful(Done)) + } } yield status.map(_._1) } } yield application @@ -117,22 +131,39 @@ class ApplicationsService @Inject() ( def applicationStatus( )(implicit hat: HatServer, user: HatUser, - requestHeader: RequestHeader): Future[Seq[HatApplication]] = + requestHeader: RequestHeader): Future[Seq[HatApplication]] = { + logger.info(s"Fetching application status") cache .get[Seq[HatApplication]](s"apps:${hat.domain}") .flatMap { - case Some(applications) => Future.successful(applications) + case Some(applications) => + logger.debug("Application was found in the cache") + Future.successful(applications) case None => + logger.debug("Application was not found in the cache. Fetching application") for { - apps <- trustedApplicationProvider.applications // potentially caching - setup <- applicationSetupStatus()(hat.db) // database - statuses <- Future.sequence(apps.map(refetchApplicationsStatus(_, setup))) + apps <- { + logger.debug("Fetching applications") + trustedApplicationProvider.applications // potentially caching + } + setup <- { + logger.debug("Fetching application setup status") + applicationSetupStatus()(hat.db) // database + } + statuses <- { + logger.debug(s"Refreshing application status") + Future.sequence(apps.map(refetchApplicationsStatus(_, setup))) + } apps = statuses.map(_._1) - _ <- if (statuses.forall(_._2)) - cache.set(s"apps:${hat.domain}", apps, applicationsCacheDuration) - else Future.successful(Done) + _ <- { + logger.debug("Attempting to save applications in cache") + if (statuses.forall(_._2)) + cache.set(s"apps:${hat.domain}", apps, applicationsCacheDuration) + else Future.successful(Done) + } } yield apps } + } private def refetchApplicationsStatus( app: Application, @@ -291,6 +322,7 @@ class ApplicationsService @Inject() ( )(implicit hat: HatServer, user: HatUser, requestHeader: RequestHeader): Future[HatApplication] = { + logger.info(s"Setting app application: ${application.application.id}") val appSetup = for { // Create and enable the data debit _ <- enableAssociatedDataDebit(application) diff --git a/hat/app/org/hatdex/hat/api/service/applications/TrustedApplicationProviderDex.scala b/hat/app/org/hatdex/hat/api/service/applications/TrustedApplicationProviderDex.scala index 2b3517b27..120036831 100644 --- a/hat/app/org/hatdex/hat/api/service/applications/TrustedApplicationProviderDex.scala +++ b/hat/app/org/hatdex/hat/api/service/applications/TrustedApplicationProviderDex.scala @@ -1,8 +1,8 @@ package org.hatdex.hat.api.service.applications import io.dataswift.models.hat.applications.Application -import org.hatdex.dex.apiV2.DexClient -import org.hatdex.dex.apiV2.Errors.ApiException +import org.hatdex.dex.apiV3.DexClient +import org.hatdex.dex.apiV3.Errors.ApiException import org.hatdex.hat.api.service.RemoteExecutionContext import play.api.cache.AsyncCacheApi import play.api.libs.ws.WSClient @@ -21,12 +21,8 @@ class TrustedApplicationProviderDex @Inject() ( private val logger = Logger(this.getClass) - private val dexClient = new DexClient( - wsClient, - configuration.underlying.getString("exchange.address"), - configuration.underlying.getString("exchange.scheme"), - "v1.1" - ) + private val dexAddress = configuration.underlying.getString("exchange.address") + private val dexClient = new DexClient(wsClient, dexAddress) private val applicationsCacheDuration = configuration.get[FiniteDuration]("application-cache-ttl") @@ -38,7 +34,7 @@ class TrustedApplicationProviderDex @Inject() ( "apps:dexApplications", applicationsCacheDuration ) { - dexClient.applications(includeUnpublished = includeUnpublished) + dexClient.applications(Some(includeUnpublished)) } def application(id: String): Future[Option[Application]] = diff --git a/hat/app/org/hatdex/hat/api/service/richData/DataDebitService.scala b/hat/app/org/hatdex/hat/api/service/richData/DataDebitService.scala index 502a2765b..35170e955 100644 --- a/hat/app/org/hatdex/hat/api/service/richData/DataDebitService.scala +++ b/hat/app/org/hatdex/hat/api/service/richData/DataDebitService.scala @@ -34,8 +34,8 @@ import org.hatdex.hat.resourceManagement.HatServer import org.hatdex.hat.utils.FutureTransformations import org.hatdex.libs.dal.HATPostgresProfile.api._ import org.joda.time.LocalDateTime -import play.api.Logger import play.api.libs.json._ +import play.api.{ Configuration, Logger } import java.sql.SQLException import java.util.UUID @@ -44,11 +44,13 @@ import scala.concurrent.Future import scala.util.Success class DataDebitService @Inject() ( - userService: UserService + userService: UserService, + config: Configuration )(implicit val ec: RemoteExecutionContext) { import RichDataJsonFormats._ val logger: Logger = Logger(this.getClass) + val dexAddress: String = config.get[String]("exchange.address") def createDataDebit( key: String, @@ -310,11 +312,11 @@ class DataDebitService @Inject() ( else dd.requestClientName, requestClientLogoUrl = if (dd.requestClientName.isEmpty) - "https://dex.hubofallthings.com/assets//images/dex.png" + s"$dexAddress/assets/images/dex.png" else dd.requestClientLogoUrl, requestClientUrl = if (dd.requestClientName.isEmpty) - "https://dex.hubofallthings.com/" + dexAddress else dd.requestClientUrl ) ) diff --git a/hat/app/org/hatdex/hat/modules/AppModule.scala b/hat/app/org/hatdex/hat/modules/AppModule.scala index 09eb0f181..81f1be65f 100644 --- a/hat/app/org/hatdex/hat/modules/AppModule.scala +++ b/hat/app/org/hatdex/hat/modules/AppModule.scala @@ -23,11 +23,9 @@ class AppModule extends ScalaModule { configuration: Configuration, wsClient: WSClient): AdjudicatorClient = { val adjudicatorAddress = configuration.get[String]("adjudicator.address") - val adjudicatorScheme = configuration.get[String]("adjudicator.scheme") - val adjudicatorEndpoint = s"$adjudicatorScheme$adjudicatorAddress" val adjudicatorSharedSecret = configuration.get[String]("adjudicator.sharedSecret") new AdjudicatorWsClient( - adjudicatorEndpoint, + adjudicatorAddress, JwtSecretKey(adjudicatorSharedSecret), wsClient ) diff --git a/hat/app/org/hatdex/hat/resourceManagement/HatKeyProvider.scala b/hat/app/org/hatdex/hat/resourceManagement/HatKeyProvider.scala index 185522a9a..60e5bd5f1 100644 --- a/hat/app/org/hatdex/hat/resourceManagement/HatKeyProvider.scala +++ b/hat/app/org/hatdex/hat/resourceManagement/HatKeyProvider.scala @@ -112,7 +112,7 @@ class HatKeyProviderConfig @Inject() (configuration: Configuration) extends HatK readRsaPrivateKey(confPrivateKey) } getOrElse { Future.failed( - new HatServerDiscoveryException(s"Private Key for $hat not found") + new HatServerDiscoveryException(s"Private Key for $hat not found in the configuration") ) } @@ -154,18 +154,20 @@ class HatKeyProviderMilliner @Inject() ( def privateKey( hat: String - )(implicit ec: ExecutionContext): Future[RSAPrivateKey] = + )(implicit ec: ExecutionContext): Future[RSAPrivateKey] = { + logger.info(s"Fetching private Key for hat: $hat") getHatSignup(hat) flatMap { signup => logger.debug( s"Received signup info, parsing private key ${signup.keys.map(_.privateKey)}" ) readRsaPrivateKey(signup.keys.get.privateKey) } recoverWith { - case _ => - Future.failed( - new HatServerDiscoveryException(s"Private Key for $hat not found") - ) - } + case _ => + Future.failed( + new HatServerDiscoveryException(s"Private Key for $hat not found") + ) + } + } def ownerEmail(hat: String)(implicit ec: ExecutionContext): Future[String] = getHatSignup(hat) map { signup => diff --git a/hat/app/org/hatdex/hat/resourceManagement/MillinerHatSignup.scala b/hat/app/org/hatdex/hat/resourceManagement/MillinerHatSignup.scala index 84a1fcaef..a0300ae3b 100644 --- a/hat/app/org/hatdex/hat/resourceManagement/MillinerHatSignup.scala +++ b/hat/app/org/hatdex/hat/resourceManagement/MillinerHatSignup.scala @@ -38,17 +38,9 @@ trait MillinerHatSignup { val logger: Logger val ws: WSClient val configuration: Configuration - val schema: String = - configuration.get[String]("resourceManagement.millinerAddress") match { - case address if address.startsWith("https") => "https://" - case address if address.startsWith("http") => "http://" - case _ => "https://" - } val millinerAddress: String = configuration .get[String]("resourceManagement.millinerAddress") - .stripPrefix("http://") - .stripPrefix("https://") val hatSharedSecret: String = configuration.get[String]("resourceManagement.hatSharedSecret") @@ -60,8 +52,7 @@ trait MillinerHatSignup { // Cache the signup information for subsequent calls (For private/public key and database details) cache.getOrElseUpdate[HatSignup](s"configuration:$hatAddress") { val request: WSRequest = ws - .url(s"$schema$millinerAddress/api/manage/configuration/$hatAddress") - .withVirtualHost(millinerAddress) + .url(s"$millinerAddress/api/manage/configuration/$hatAddress") .withHttpHeaders( "Accept" -> "application/json", "X-Auth-Token" -> hatSharedSecret diff --git a/hat/app/org/hatdex/hat/resourceManagement/actors/HatServerActor.scala b/hat/app/org/hatdex/hat/resourceManagement/actors/HatServerActor.scala index aaca668a5..aa1292396 100644 --- a/hat/app/org/hatdex/hat/resourceManagement/actors/HatServerActor.scala +++ b/hat/app/org/hatdex/hat/resourceManagement/actors/HatServerActor.scala @@ -126,6 +126,7 @@ class HatServerActor @Inject() ( } private def server(hat: String): Future[HatServer] = { + log.debug(s"fetching hat keys for hat: $hat") val server = for { privateKey <- hatKeyProvider.privateKey(hat) publicKey <- hatKeyProvider.publicKey(hat) diff --git a/hat/app/org/hatdex/hat/utils/DataswiftServiceConfig.scala b/hat/app/org/hatdex/hat/utils/DataswiftServiceConfig.scala index 5f2b5bc91..caa859f6d 100644 --- a/hat/app/org/hatdex/hat/utils/DataswiftServiceConfig.scala +++ b/hat/app/org/hatdex/hat/utils/DataswiftServiceConfig.scala @@ -3,13 +3,7 @@ package org.hatdex.hat.utils case class DataswiftServiceConfig( name: String, host: String, - //apiVersion: String, - secure: Boolean, path: Option[String]) { - lazy val address: String = { - val scheme = if (secure) "https" else "http" - - s"$scheme://$host${path.getOrElse("")}" - } + lazy val address: String = s"$host${path.getOrElse("")}" } diff --git a/hat/conf/application.conf b/hat/conf/application.conf index 96af2a3d6..10484adef 100644 --- a/hat/conf/application.conf +++ b/hat/conf/application.conf @@ -96,7 +96,7 @@ resourceManagement { serverProvisioningTimeout = 5 seconds serverIdleTimeout = 3 minutes serverIdleTimeout = ${?RESOURCE_MGMT_SERVER_IDLE_TIMEOUT} - millinerAddress = "https://eu1.hubofallthings.net" + millinerAddress = "https://eu1.hubofallthings.net" #"https://one.dataswift.net/store" millinerAddress = ${?MILLINER_ADDRESS} hatSharedSecret = "" hatSharedSecret = ${?HAT_SHARED_SECRET} @@ -119,24 +119,18 @@ storage { } exchange { - admin = ["systems@hatdex.org"] + admin = ["systems@dataswift.net"] admin = [${?HAT_ADMIN_EMAIL}] - address = "dex.hubofallthings.com" + address = "https://dex.hubofallthings.com" #"https://one.dataswift.net/store" address = ${?DEX_ADDRESS} - scheme = "https://" retryLimit = 10 retryTime = 30 minutes - batchSize = 20 - storage { - collectionPeriod = 1 hour - } beta = ${?HAT_BETA} } adjudicator { - address = "contracts.dataswift.dev" + address = "https://contracts.dataswift.dev" address = ${?ADJUDICATOR_ADDRESS} - scheme = "https://" sharedSecret = "SyqogHEnVPVUJkCE6fS43/KBl0hrmNUcsMhTPymD1Oc=" sharedSecret = ${?ADJUDICATOR_SHARED_SECRET} } @@ -144,9 +138,8 @@ adjudicator { pdaAccountRegistry { verificationCallback { name = "hatters" - host = "auth.dataswift.dev" + host = "https://auth.dataswift.dev" #"https://one.dataswift.net/auth" host = ${?PDA_REGISTRY_HOST} - secure = true path = "/api/services/daas/claim" } } @@ -160,6 +153,10 @@ mailer { mock = ${?MAILER_MOCK} } +play-prometheus-filters.exclude.paths = ["/metrics"] +# play-prometheus-filters.exclude.paths = ["/test"] +play-prometheus-filters.metric.resolution = "milliseconds" + include "she.conf" include "phata.conf" diff --git a/hat/conf/play.conf b/hat/conf/play.conf index 63f09c702..aa61bdf08 100644 --- a/hat/conf/play.conf +++ b/hat/conf/play.conf @@ -35,36 +35,6 @@ play { } filters { - cors { - pathPrefixes = [ - "/users", - "/system/update", - "/api/v2", - "/publickey" - ] - allowedOrigins = [ - ${?CORS_ALLOWED_ORIGIN_1}, - ${?CORS_ALLOWED_ORIGIN_2}, - ${?CORS_ALLOWED_ORIGIN_3}, - ${?CORS_ALLOWED_ORIGIN_4} - ] - allowedHttpHeaders = ["Accept", - "Accept-Encoding", - "Accept-Language", - "Authorization", - "Content-Type", - "Host", - "Origin", - "Referer", - "User-Agent", - "X-Requested-With", - "X-Auth-Token", - ${?CORS_ALLOWED_HTTP_HEADER_1}] - exposedHeaders = ["X-Auth-Token"] - preflightMaxAge = 3 days, - supportsCredentials = true // Allows for cookies to work with sticky loadbalancer sessions - } - csrf.header.bypassHeaders { X-Requested-With = "*" Csrf-Token = "nocheck" @@ -86,7 +56,6 @@ play { enabled = [ "com.github.stijndehaes.playprometheusfilters.filters.StatusAndRouteLatencyAndCounterFilter", "org.hatdex.hat.utils.LoggingFilter", - "play.filters.cors.CORSFilter", "play.filters.gzip.GzipFilter" ] } diff --git a/hat/conf/regions.conf b/hat/conf/regions.conf index b37e399d3..2571a19f7 100644 --- a/hat/conf/regions.conf +++ b/hat/conf/regions.conf @@ -5,4 +5,3 @@ datacenter { deployedGeneralRegion = "EU" deployedGeneralRegion = ${?DEPLOYED_GENERAL_REGION} } - diff --git a/hat/conf/routes b/hat/conf/routes index 0f7348761..b82efe278 100644 --- a/hat/conf/routes +++ b/hat/conf/routes @@ -16,6 +16,7 @@ PUT /users/user/:userId/update org.hatdex.hat. # SYSTEM routes GET /system/update org.hatdex.hat.api.controllers.SystemStatus.update() +GET /healthz org.hatdex.hat.api.controllers.SystemStatus.healthReport() # AUTHENTICATION routes GET /control/v2/auth/hatlogin org.hatdex.hat.api.controllers.Authentication.hatLogin(name: String, redirect: String) diff --git a/hat/conf/v26.routes b/hat/conf/v26.routes index 1fb5165f2..43d5effd0 100644 --- a/hat/conf/v26.routes +++ b/hat/conf/v26.routes @@ -1,5 +1,7 @@ GET /system/status org.hatdex.hat.api.controllers.SystemStatus.status() +GET /healthz org.hatdex.hat.api.controllers.SystemStatus.healthReport() + GET /phata/profile org.hatdex.hat.phata.controllers.Phata.profile POST /report-frontend-action org.hatdex.hat.api.controllers.LogController.logFrontendAction DELETE /system/destroy-cache org.hatdex.hat.api.controllers.SystemStatus.destroyCache() diff --git a/hat/test/com/github/stijndehaes/playprometheusfilters/filters/PrometheusFiltersSpec.scala b/hat/test/com/github/stijndehaes/playprometheusfilters/filters/PrometheusFiltersSpec.scala index 96193a10f..f206fe13c 100644 --- a/hat/test/com/github/stijndehaes/playprometheusfilters/filters/PrometheusFiltersSpec.scala +++ b/hat/test/com/github/stijndehaes/playprometheusfilters/filters/PrometheusFiltersSpec.scala @@ -23,7 +23,9 @@ class PrometheusFiltersSpec extends BaseSpec { implicit val system: ActorSystem = ActorSystem() implicit val materializer: ActorMaterializer = ActorMaterializer() - private val noExclusionsConfig = Configuration("play-prometheus-filters.exclude.paths" -> Nil) + private val noExclusionsConfig = Configuration( + "play-prometheus-filters.exclude.paths" -> Nil, + "play-prometheus-filters.metric.resolution" -> "milliseconds") "LatencyFilter" should "Measure the latency" in { // I needed to have an independnt collector per test, otherwise they complain about having duplicate labels. @@ -70,7 +72,7 @@ class PrometheusFiltersSpec extends BaseSpec { metrics must have size 1 val samples = metrics.get(0).samples - samples.get(0).value mustBe 1.0 + samples.get(0).value mustBe 0.0 samples.get(0).labelValues must have size expectedLabelCount samples.get(0).labelValues.get(0) mustBe expectedMethod samples.get(0).labelValues.get(1) mustBe expectedStatus diff --git a/hat/test/org/hatdex/hat/api/HATTestContext.scala b/hat/test/org/hatdex/hat/api/HATTestContext.scala index ab4284872..ffe39cf68 100644 --- a/hat/test/org/hatdex/hat/api/HATTestContext.scala +++ b/hat/test/org/hatdex/hat/api/HATTestContext.scala @@ -24,7 +24,6 @@ package org.hatdex.hat.api -import akka.Done import akka.stream.Materializer import com.amazonaws.services.s3.AmazonS3 import com.atlassian.jwt.core.keys.KeyUtils diff --git a/hat/test/org/hatdex/hat/api/service/FileManagerS3Mock.scala b/hat/test/org/hatdex/hat/api/service/FileManagerS3Mock.scala index 10d5c39e9..7aace0c68 100644 --- a/hat/test/org/hatdex/hat/api/service/FileManagerS3Mock.scala +++ b/hat/test/org/hatdex/hat/api/service/FileManagerS3Mock.scala @@ -23,9 +23,15 @@ */ package org.hatdex.hat.api.service - +import com.amazonaws.auth.{ AWSStaticCredentialsProvider, BasicAWSCredentials } import com.amazonaws.services.s3.{ AmazonS3, AmazonS3ClientBuilder } class FileManagerS3Mock { - val mockS3client: AmazonS3 = AmazonS3ClientBuilder.defaultClient() + private val awsCreds = new BasicAWSCredentials("testAwsAccessKey", "testAwsSecret") + val mockS3client: AmazonS3 = + AmazonS3ClientBuilder + .standard() + .withRegion("eu-west-1") + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .build() } diff --git a/hat/test/org/hatdex/hat/api/service/FileManagerS3Spec.scala b/hat/test/org/hatdex/hat/api/service/FileManagerS3Spec.scala index 41a8884ed..701119e8f 100644 --- a/hat/test/org/hatdex/hat/api/service/FileManagerS3Spec.scala +++ b/hat/test/org/hatdex/hat/api/service/FileManagerS3Spec.scala @@ -25,14 +25,14 @@ package org.hatdex.hat.api.service import org.hatdex.hat.api.HATTestContext -import org.scalatest.time.{ Millis, Second, Span } +import org.scalatest.time.{ Millis, Seconds, Span } import scala.concurrent.Future class FileManagerS3Spec extends HATTestContext { implicit val defaultPatience: PatienceConfig = - PatienceConfig(timeout = Span(1, Second), interval = Span(50, Millis)) + PatienceConfig(timeout = Span(3, Seconds), interval = Span(50, Millis)) "The `getUploadUrl` method" should "return a signed url for a provided key" in { val fileManager = application.injector.instanceOf[FileManager] diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1db2450df..bd0f338b3 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -34,8 +34,8 @@ object Dependencies { object DsLib { private object Version { val DsAdjudicator = "0.2.0" - val DsBackend = "2.5.0" - val DsDexClient = "3.3.1" + val DsBackend = "2.5.5" + val DsDexClient = "3.4.3" val DsSilhouette = "5.3.0" val DsSlickPostgresDriver = "0.1.2" } @@ -55,8 +55,8 @@ object Dependencies { private object Version { val AlpakkaAwsLambda = "1.1.2" val CirceConfig = "0.8.0" - val PrettyTime = "5.0.0.Final" - val ScalaTestplusMockito = "3.2.10.0" + val PrettyTime = "5.0.2.Final" + val ScalaTestplusMockito = "3.2.9.0" } object LocalThirdParty { diff --git a/project/build.properties b/project/build.properties index 3b1d9b118..e680b0cc7 100644 --- a/project/build.properties +++ b/project/build.properties @@ -22,4 +22,4 @@ # 2 / 2017 # -sbt.version=1.5.3 +sbt.version=1.5.5 diff --git a/project/plugins.sbt b/project/plugins.sbt index d4d6fb834..0113fb047 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,8 +2,8 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.2") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.7") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.2.7") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.2") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.9.1")