From 5bf834f5af524e6cfb927cb98e7e2c4579508192 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Thu, 21 Apr 2022 06:57:21 +0100 Subject: [PATCH 01/35] Make trigger fire regardless. For TESTING --- .../org/hatdex/hat/she/controllers/FunctionManager.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala index b783ef97c..2d3e053cd 100644 --- a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala +++ b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala @@ -131,7 +131,7 @@ class FunctionManager @Inject() ( logger.debug(s"Trigger function $function") functionService.get(function).flatMap { maybeFunction => maybeFunction.map { - case c: FunctionConfiguration if c.status.available && c.status.enabled => + case c: FunctionConfiguration => //if c.status.available && c.status.enabled => functionExecutionDispatcher .trigger(request.dynamicEnvironment.domain, c, useAll)(ec) .map(_ => Ok(Json.toJson(SuccessResponse("Function Executed")))) @@ -150,7 +150,7 @@ class FunctionManager @Inject() ( ) ) } - case FunctionConfiguration( +/* case FunctionConfiguration( _, _, _, @@ -185,7 +185,7 @@ class FunctionManager @Inject() ( ) ) ) - ) + )*/ } getOrElse { Future.successful( NotFound( From 95aaa73cce2a35de320631fa596d2b859d5bfcb2 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 12:41:20 +0100 Subject: [PATCH 02/35] Changed SHE Function credentials provider to DefaultCredentialsProvider --- .../org/hatdex/hat/she/models/LambdaFunctionExecutable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 941b3f71f..4564331d2 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -35,7 +35,7 @@ import org.hatdex.hat.api.service.RemoteExecutionContext import org.joda.time.DateTime import play.api.libs.json.{ Format, Json } import play.api.{ Configuration, Logger } -import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.lambda.LambdaAsyncClient @@ -166,7 +166,7 @@ class AwsLambdaExecutor @Inject() ( LambdaAsyncClient .builder() .region(Region.of(configuration.get[String]("she.aws.region"))) - .credentialsProvider(ContainerCredentialsProvider.builder().build()) + .credentialsProvider(DefaultCredentialsProvider.create()) .build() actorSystem.registerOnTermination(lambdaClient.close()) From ea9e6fcc98146ae6bba1538bbe087cc51db6801c Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 13:14:06 +0100 Subject: [PATCH 03/35] Changed SHE Function credentials provider to StsAssumeRoleWithWebIdentityCredentialsProvider --- .../org/hatdex/hat/she/models/LambdaFunctionExecutable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 4564331d2..da816a62a 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -35,7 +35,7 @@ import org.hatdex.hat.api.service.RemoteExecutionContext import org.joda.time.DateTime import play.api.libs.json.{ Format, Json } import play.api.{ Configuration, Logger } -import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleWithWebIdentityCredentialsProvider import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.lambda.LambdaAsyncClient @@ -166,7 +166,7 @@ class AwsLambdaExecutor @Inject() ( LambdaAsyncClient .builder() .region(Region.of(configuration.get[String]("she.aws.region"))) - .credentialsProvider(DefaultCredentialsProvider.create()) + .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().build()) .build() actorSystem.registerOnTermination(lambdaClient.close()) From 46a5a6c4ed08522bb53b329b5591739ca2bae03e Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 20:17:44 +0100 Subject: [PATCH 04/35] Added Sts ref from updated scala-tools-common --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 6e4e03536..3208d63d4 100644 --- a/build.sbt +++ b/build.sbt @@ -29,6 +29,7 @@ lazy val hat = project DsLib.SilhouettePersistence, DsLib.SlickPostgresDriver, Lib.AwsV1Sdk, + Lib.AwsV1SdkSts, Lib.BouncyCastle, Lib.Ficus, Lib.Guard, From db5e46aed0513466365d60337d2735ffc5e308c0 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 20:44:24 +0100 Subject: [PATCH 05/35] Version bump scala-tools-common --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 8b21b1ee9..3237b72ac 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.12") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.13") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From bf22ba4a0e405231159110e65ff0d3cf96032fab Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 21:36:13 +0100 Subject: [PATCH 06/35] Version bump scala-tools-common --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 3237b72ac..f9a3d7833 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.13") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.14") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From 100f32280dbf389aebfbd87d90c47d9fb3c59c6e Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 21:42:07 +0100 Subject: [PATCH 07/35] Version bump scala-tools-common due to bucket region fix --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index f9a3d7833..e403a9e91 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.14") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.15") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From 388b5d652922d6c65b582f33a569b97fe5d170da Mon Sep 17 00:00:00 2001 From: Piotr Perzyna Date: Fri, 22 Apr 2022 22:52:51 +0200 Subject: [PATCH 08/35] Update plugins.sbt --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index e403a9e91..680c6aabf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.15") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.16") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From eea12592e50c23103a823597f713b355767d152c Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 22:35:45 +0100 Subject: [PATCH 09/35] Version bump scala-tools-common. STS v2 is actually in use --- build.sbt | 2 +- project/plugins.sbt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 3208d63d4..c8f76dc90 100644 --- a/build.sbt +++ b/build.sbt @@ -29,7 +29,7 @@ lazy val hat = project DsLib.SilhouettePersistence, DsLib.SlickPostgresDriver, Lib.AwsV1Sdk, - Lib.AwsV1SdkSts, + Lib.AwsV2SdkSts, Lib.BouncyCastle, Lib.Ficus, Lib.Guard, diff --git a/project/plugins.sbt b/project/plugins.sbt index 680c6aabf..776a9d4d4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.16") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.17") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From b7ba4e4a33cfc9b1b35c4ca68f3306b45405ab36 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 22:47:29 +0100 Subject: [PATCH 10/35] Reduce version AWS STS for SDKv2, due to PDA lib compatibilities --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 776a9d4d4..6f4a44375 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.17") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.18") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From c7cd5d09d9cfb5524e077fbc1c1edcfce942ec1a Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Fri, 22 Apr 2022 23:53:08 +0100 Subject: [PATCH 11/35] Test passed --- project/BasicSettings.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/project/BasicSettings.scala b/project/BasicSettings.scala index 10586322c..89066a67d 100644 --- a/project/BasicSettings.scala +++ b/project/BasicSettings.scala @@ -72,4 +72,10 @@ object BasicSettings extends AutoPlugin { // and https://github.com/travis-ci/travis-ci/issues/3775 javaOptions += "-Xmx1G" ) + + object autoImport { + // Do Nothing + // Addet by Terry 20220422 + // Not sure why this block is missing and how you got HAT to compile in the first place? + } } From f3b952a9874bb26278ef8d6958481e0b5b4a0810 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 00:20:14 +0100 Subject: [PATCH 12/35] Remove credentials provider. (Is it really that simple) --- .../hatdex/hat/she/models/LambdaFunctionExecutable.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index da816a62a..bbeec84db 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -43,6 +43,7 @@ import software.amazon.awssdk.services.lambda.model.{ InvokeRequest, InvokeRespo import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } +import software.amazon.awssdk.services.sts.StsClient class LambdaFunctionExecutable( id: String, @@ -162,11 +163,15 @@ class AwsLambdaExecutor @Inject() ( implicit private val materializer: Materializer = ActorMaterializer() + private val region = Region.of(configuration.get[String]("she.aws.region")) + + implicit private val stsClient = StsClient.builder().region(this.region).build(); + implicit private val lambdaClient: LambdaAsyncClient = LambdaAsyncClient .builder() - .region(Region.of(configuration.get[String]("she.aws.region"))) - .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().build()) + .region(this.region) +// .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).build()) .build() actorSystem.registerOnTermination(lambdaClient.close()) From abddf6c1b2db887b27b6c9ba5d130b7f3f9fdefe Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 10:41:56 +0100 Subject: [PATCH 13/35] Added RoleArn config for SHE executor --- .../hatdex/hat/she/models/LambdaFunctionExecutable.scala | 8 ++++++-- hat/conf/she.conf | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index bbeec84db..3e615ffa4 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -44,6 +44,8 @@ import software.amazon.awssdk.services.lambda.model.{ InvokeRequest, InvokeRespo import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest +import java.util.UUID class LambdaFunctionExecutable( id: String, @@ -164,14 +166,16 @@ class AwsLambdaExecutor @Inject() ( implicit private val materializer: Materializer = ActorMaterializer() private val region = Region.of(configuration.get[String]("she.aws.region")) + private val roleArn = configuration.get[String]("she.aws.roleArn") implicit private val stsClient = StsClient.builder().region(this.region).build(); - + private val assumeRoleRequest = AssumeRoleWithWebIdentityRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).build(); + implicit private val lambdaClient: LambdaAsyncClient = LambdaAsyncClient .builder() .region(this.region) -// .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).build()) + .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) .build() actorSystem.registerOnTermination(lambdaClient.close()) diff --git a/hat/conf/she.conf b/hat/conf/she.conf index 7049c0c67..6b879a09b 100644 --- a/hat/conf/she.conf +++ b/hat/conf/she.conf @@ -5,6 +5,8 @@ she { region = ${?AWS_REGION} logs = "Tail" mock = false + roleArn = "" + roleArn = ${?SHE_ROLE_ARN} } executionDispatcher { maxHats = 1000 From 7634279544c697cb34a5729cb4732f7eab1fddbe Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 11:31:29 +0100 Subject: [PATCH 14/35] Added logger to see exactly what killed the SHE --- .../hatdex/hat/she/service/FunctionExecutionDispatcher.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index 27b5fa18d..fd4734fb0 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -202,7 +202,10 @@ class FunctionExecutionTriggerHandler @Inject() ( functionService .run(conf, conf.status.lastExecution, useAll)(hatServer) .recover { - case e => throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) + case e => { + logger.error(e.getMessage(), e) + throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) + } } } getOrElse { Future.failed( From 77e5efa82b55db727ae3a7bf3d3238270e7ab53a Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 12:26:13 +0100 Subject: [PATCH 15/35] There is another point to add for Trigger Regardless --- .../hat/she/service/FunctionExecutionDispatcher.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index fd4734fb0..9006f8dff 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -75,7 +75,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, true, Some(lastExecution), None) + FunctionStatus(true, _, Some(lastExecution), None) ) if lastExecution.isBefore(DateTime.now().minus(period)) => true case FunctionConfiguration(_, @@ -83,7 +83,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, true, Some(lastExecution), Some(started)) + FunctionStatus(true, _, Some(lastExecution), Some(started)) ) if lastExecution.isBefore(DateTime.now().minus(period)) && started .isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => @@ -93,7 +93,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, true, None, None) + FunctionStatus(true, _, None, None) ) => true // no execution recoded yet case FunctionConfiguration(_, @@ -101,7 +101,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, true, None, Some(started)) + FunctionStatus(true, _, None, Some(started)) ) if started.isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => true // no successful execution, current one timed out case FunctionConfiguration(_, @@ -109,7 +109,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerIndividual(), _, - FunctionStatus(true, true, _, _) + FunctionStatus(true, _, _, _) ) => true case _ => false // in all other cases, do not trigger From 1f6a5ca0be85c1c0f95634129da7974bb52e061f Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 15:08:41 +0100 Subject: [PATCH 16/35] Changed AWS creds provider --- .../org/hatdex/hat/she/controllers/FunctionManager.scala | 4 ++-- .../hatdex/hat/she/models/LambdaFunctionExecutable.scala | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala index 2d3e053cd..0f8c1784f 100644 --- a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala +++ b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala @@ -150,7 +150,7 @@ class FunctionManager @Inject() ( ) ) } -/* case FunctionConfiguration( + case FunctionConfiguration( _, _, _, @@ -185,7 +185,7 @@ class FunctionManager @Inject() ( ) ) ) - )*/ + ) } getOrElse { Future.successful( NotFound( diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 3e615ffa4..e0bac10e6 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -35,7 +35,6 @@ import org.hatdex.hat.api.service.RemoteExecutionContext import org.joda.time.DateTime import play.api.libs.json.{ Format, Json } import play.api.{ Configuration, Logger } -import software.amazon.awssdk.services.sts.auth.StsAssumeRoleWithWebIdentityCredentialsProvider import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.lambda.LambdaAsyncClient @@ -44,8 +43,9 @@ import software.amazon.awssdk.services.lambda.model.{ InvokeRequest, InvokeRespo import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } import software.amazon.awssdk.services.sts.StsClient -import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest import java.util.UUID +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider class LambdaFunctionExecutable( id: String, @@ -169,13 +169,13 @@ class AwsLambdaExecutor @Inject() ( private val roleArn = configuration.get[String]("she.aws.roleArn") implicit private val stsClient = StsClient.builder().region(this.region).build(); - private val assumeRoleRequest = AssumeRoleWithWebIdentityRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).build(); + private val assumeRoleRequest = AssumeRoleRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).build(); implicit private val lambdaClient: LambdaAsyncClient = LambdaAsyncClient .builder() .region(this.region) - .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) + .credentialsProvider(StsAssumeRoleCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) .build() actorSystem.registerOnTermination(lambdaClient.close()) From d3af3d4b1edaa10622f806b352bb75860db21303 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 15:47:23 +0100 Subject: [PATCH 17/35] Revert --- .../hat/she/service/FunctionExecutionDispatcher.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index 9006f8dff..fd4734fb0 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -75,7 +75,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, _, Some(lastExecution), None) + FunctionStatus(true, true, Some(lastExecution), None) ) if lastExecution.isBefore(DateTime.now().minus(period)) => true case FunctionConfiguration(_, @@ -83,7 +83,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, _, Some(lastExecution), Some(started)) + FunctionStatus(true, true, Some(lastExecution), Some(started)) ) if lastExecution.isBefore(DateTime.now().minus(period)) && started .isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => @@ -93,7 +93,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, _, None, None) + FunctionStatus(true, true, None, None) ) => true // no execution recoded yet case FunctionConfiguration(_, @@ -101,7 +101,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, _, None, Some(started)) + FunctionStatus(true, true, None, Some(started)) ) if started.isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => true // no successful execution, current one timed out case FunctionConfiguration(_, @@ -109,7 +109,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerIndividual(), _, - FunctionStatus(true, _, _, _) + FunctionStatus(true, true, _, _) ) => true case _ => false // in all other cases, do not trigger From 58d7170bf5634f8dc8e02541fabd68f443cfc971 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 16:45:45 +0100 Subject: [PATCH 18/35] Add AWS_WEB_IDENTITY_TOKEN_FILE --- .../hat/she/models/LambdaFunctionExecutable.scala | 8 ++++---- .../hat/she/service/FunctionExecutionDispatcher.scala | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index e0bac10e6..a4278c76d 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -44,8 +44,8 @@ import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } import software.amazon.awssdk.services.sts.StsClient import java.util.UUID -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest -import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider +import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleWithWebIdentityCredentialsProvider class LambdaFunctionExecutable( id: String, @@ -169,13 +169,13 @@ class AwsLambdaExecutor @Inject() ( private val roleArn = configuration.get[String]("she.aws.roleArn") implicit private val stsClient = StsClient.builder().region(this.region).build(); - private val assumeRoleRequest = AssumeRoleRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).build(); + private val assumeRoleRequest = AssumeRoleWithWebIdentityRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).webIdentityToken(System.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")).build(); implicit private val lambdaClient: LambdaAsyncClient = LambdaAsyncClient .builder() .region(this.region) - .credentialsProvider(StsAssumeRoleCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) + .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) .build() actorSystem.registerOnTermination(lambdaClient.close()) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index fd4734fb0..9006f8dff 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -75,7 +75,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, true, Some(lastExecution), None) + FunctionStatus(true, _, Some(lastExecution), None) ) if lastExecution.isBefore(DateTime.now().minus(period)) => true case FunctionConfiguration(_, @@ -83,7 +83,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, true, Some(lastExecution), Some(started)) + FunctionStatus(true, _, Some(lastExecution), Some(started)) ) if lastExecution.isBefore(DateTime.now().minus(period)) && started .isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => @@ -93,7 +93,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, true, None, None) + FunctionStatus(true, _, None, None) ) => true // no execution recoded yet case FunctionConfiguration(_, @@ -101,7 +101,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, true, None, Some(started)) + FunctionStatus(true, _, None, Some(started)) ) if started.isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => true // no successful execution, current one timed out case FunctionConfiguration(_, @@ -109,7 +109,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerIndividual(), _, - FunctionStatus(true, true, _, _) + FunctionStatus(true, _, _, _) ) => true case _ => false // in all other cases, do not trigger From 01235d22c34b4a254d4175f311fc8d8db7ac06e1 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 17:30:52 +0100 Subject: [PATCH 19/35] Function finder wrong match --- .../hat/she/service/FunctionExecutionDispatcher.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index 9006f8dff..9f29d0c87 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -75,7 +75,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, _, Some(lastExecution), None) + FunctionStatus(_, true, Some(lastExecution), None) ) if lastExecution.isBefore(DateTime.now().minus(period)) => true case FunctionConfiguration(_, @@ -83,7 +83,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(true, _, Some(lastExecution), Some(started)) + FunctionStatus(_, true, Some(lastExecution), Some(started)) ) if lastExecution.isBefore(DateTime.now().minus(period)) && started .isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => @@ -93,7 +93,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, _, None, None) + FunctionStatus(_, true, None, None) ) => true // no execution recoded yet case FunctionConfiguration(_, @@ -101,7 +101,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(true, _, None, Some(started)) + FunctionStatus(_, true, None, Some(started)) ) if started.isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => true // no successful execution, current one timed out case FunctionConfiguration(_, @@ -109,7 +109,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerIndividual(), _, - FunctionStatus(true, _, _, _) + FunctionStatus(_, true, _, _) ) => true case _ => false // in all other cases, do not trigger From e639248341580b4ffd8344588198c2d53e48f911 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sat, 23 Apr 2022 18:04:48 +0100 Subject: [PATCH 20/35] Print Stack trace the thing --- hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala index 0f8c1784f..17ee9ffda 100644 --- a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala +++ b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala @@ -138,7 +138,7 @@ class FunctionManager @Inject() ( .recover { case e => logger.error( - s"Function $function execution errored with ${e.getMessage}", + s"Function $function execution errored with ${e.getStackTrace}", e ) InternalServerError( From 7c4bf891305e5d1657650025723b797d55024cbc Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 00:13:33 +0100 Subject: [PATCH 21/35] Removed AlpakkaAwsLambda from codes --- build.sbt | 4 +- .../hat/she/controllers/FunctionManager.scala | 2 +- .../she/models/LambdaFunctionExecutable.scala | 48 ++++++++++++------- project/Dependencies.scala | 2 +- project/plugins.sbt | 2 +- 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/build.sbt b/build.sbt index c8f76dc90..1cc859a68 100644 --- a/build.sbt +++ b/build.sbt @@ -29,7 +29,8 @@ lazy val hat = project DsLib.SilhouettePersistence, DsLib.SlickPostgresDriver, Lib.AwsV1Sdk, - Lib.AwsV2SdkSts, + Lib.AwsV2SdkLambda, + Lib.AwsV2SdkAuth, Lib.BouncyCastle, Lib.Ficus, Lib.Guard, @@ -40,7 +41,6 @@ lazy val hat = project Lib.PlayTest, Lib.PrometheusFilter, Lib.ScalaGuice, - LocalThirdParty.AlpakkaAwsLambda, LocalThirdParty.CirceConfig, LocalThirdParty.PrettyTime, DsLib.IntegrationTestCommon % Test, diff --git a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala index 17ee9ffda..0f8c1784f 100644 --- a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala +++ b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala @@ -138,7 +138,7 @@ class FunctionManager @Inject() ( .recover { case e => logger.error( - s"Function $function execution errored with ${e.getStackTrace}", + s"Function $function execution errored with ${e.getMessage}", e ) InternalServerError( diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index a4278c76d..129113f84 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -25,8 +25,6 @@ package org.hatdex.hat.she.models import akka.actor.ActorSystem -import akka.stream.alpakka.awslambda.scaladsl.AwsLambdaFlow -import akka.stream.scaladsl.{ Sink, Source } import akka.stream.{ ActorMaterializer, Materializer } import io.dataswift.models.hat.EndpointDataBundle import io.dataswift.models.hat.applications.Version @@ -39,13 +37,10 @@ import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.lambda.LambdaAsyncClient import software.amazon.awssdk.services.lambda.model.{ InvokeRequest, InvokeResponse } +import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } -import software.amazon.awssdk.services.sts.StsClient -import java.util.UUID -import software.amazon.awssdk.services.sts.model.AssumeRoleWithWebIdentityRequest -import software.amazon.awssdk.services.sts.auth.StsAssumeRoleWithWebIdentityCredentialsProvider class LambdaFunctionExecutable( id: String, @@ -165,17 +160,11 @@ class AwsLambdaExecutor @Inject() ( implicit private val materializer: Materializer = ActorMaterializer() - private val region = Region.of(configuration.get[String]("she.aws.region")) - private val roleArn = configuration.get[String]("she.aws.roleArn") - - implicit private val stsClient = StsClient.builder().region(this.region).build(); - private val assumeRoleRequest = AssumeRoleWithWebIdentityRequest.builder().roleSessionName(UUID.randomUUID().toString()).roleArn(this.roleArn).webIdentityToken(System.getenv("AWS_WEB_IDENTITY_TOKEN_FILE")).build(); - implicit private val lambdaClient: LambdaAsyncClient = LambdaAsyncClient .builder() - .region(this.region) - .credentialsProvider(StsAssumeRoleWithWebIdentityCredentialsProvider.builder().stsClient(this.stsClient).refreshRequest(this.assumeRoleRequest).build()) + .region(Region.of(configuration.get[String]("she.aws.region"))) + .credentialsProvider(WebIdentityTokenFileCredentialsProvider.create()) .build() actorSystem.registerOnTermination(lambdaClient.close()) @@ -184,7 +173,33 @@ class AwsLambdaExecutor @Inject() ( request: InvokeRequest )(implicit jsonFormatter: Format[T]): Future[T] = if (mock) Future.successful(null.asInstanceOf[T]) - else + else { + val invokeResponse: InvokeResponse = lambdaClient.invoke{request}.get + invokeResponse match { + case r: InvokeResponse if r.statusCode() == 200 => + logger.debug(s"""Function responded with: + | Status: ${r.statusCode()} + | Body: ${r.payload().asUtf8String()} + | Logs: ${Option(r.logResult()).map(log => java.util.Base64.getDecoder.decode(log))} + """.stripMargin) + val jsResponse = + Json.parse(r.payload().asUtf8String()).validate[T] recover { + case e => + val message = s"Error parsing lambda response: $e" + logger.error(message) + throw DataFormatException(message) + } + Future(jsResponse.get) + case r => + val message = + s"Retrieving SHE function configuration failed: $r, ${r.payload().asUtf8String()}" + logger.error(message) + throw new ApiException(message) + } + } + + + /* Source .single(request) .via(AwsLambdaFlow(1)(lambdaClient)) @@ -209,5 +224,6 @@ class AwsLambdaExecutor @Inject() ( s"Retrieving SHE function configuration failed: $r, ${r.payload().asUtf8String()}" logger.error(message) throw new ApiException(message) - } + }*/ + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5adaed5e1..88f9cc242 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -53,7 +53,7 @@ object Dependencies { } private object Version { - val AlpakkaAwsLambda = "1.1.2" + val AlpakkaAwsLambda = "3.0.4" val CirceConfig = "0.8.0" val PrettyTime = "5.0.2.Final" val ScalaTestplusMockito = "3.2.10.0" diff --git a/project/plugins.sbt b/project/plugins.sbt index 6f4a44375..0adbb15dc 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,7 +2,7 @@ resolvers += "HAT Library Artifacts Releases" at "https://s3-eu-west-1.amazonaws addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") addSbtPlugin("com.typesafe.sbt" % "sbt-web" % "1.4.4") -addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.18") +addSbtPlugin("io.dataswift" % "sbt-scalatools-common" % "0.5.22") addSbtPlugin("org.hatdex" % "sbt-slick-postgres-generator" % "0.1.2") addSbtPlugin("org.irundaia.sbt" % "sbt-sassify" % "1.5.1") addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.3.1") From c170d74248c4bdec491e7d3f2ec0b8fd500d1d93 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 00:41:02 +0100 Subject: [PATCH 22/35] Added STS. because To use web identity tokens, the 'sts' service module must be on the class path --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 1cc859a68..8a8d470f7 100644 --- a/build.sbt +++ b/build.sbt @@ -30,6 +30,7 @@ lazy val hat = project DsLib.SlickPostgresDriver, Lib.AwsV1Sdk, Lib.AwsV2SdkLambda, + Lib.AwsV2SdkSts, Lib.AwsV2SdkAuth, Lib.BouncyCastle, Lib.Ficus, From 5504ba73548c6bab0cc88eb22512109b57da7c05 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 01:11:51 +0100 Subject: [PATCH 23/35] Execution trigger fix --- .../hatdex/hat/she/service/FunctionExecutionDispatcher.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index 9f29d0c87..d040cf4cf 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -67,7 +67,7 @@ class FunctionExecutionTriggerHandler @Inject() ( .flatMap { case Some(hatServer) => functionService - .all(active = true)(hatServer.db) + .all(active = false)(hatServer.db) // TERRY this must be true in non-debug .map( _.filter({ case FunctionConfiguration(_, From b73bf466150458fad10749591f9bb822f1fb9a8e Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 01:13:33 +0100 Subject: [PATCH 24/35] Remove common-locations by Zena Woods --- hat/conf/she.conf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hat/conf/she.conf b/hat/conf/she.conf index 6b879a09b..37b004770 100644 --- a/hat/conf/she.conf +++ b/hat/conf/she.conf @@ -62,15 +62,6 @@ she { endpoint = "insights/busy-time" experimental = true } - { - id = "common-locations" - version = "1.0.0" - baseUrl = "common-locations-dev" - baseUrl = ${?COMMON_LOCATION_URL} - namespace = "she" - endpoint = "insights/common-locations" - experimental = true - } { id = "covid19-score" version = "1.0.0" From 35dbd795814e30b89da2b2ea931072c80a4b6059 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 02:22:15 +0100 Subject: [PATCH 25/35] Remove RoleArn env. Set Akka loglevel to read from env --- .../hatdex/hat/she/controllers/FunctionManager.scala | 2 +- .../she/service/FunctionExecutionDispatcher.scala | 12 ++++++------ hat/conf/base.conf | 1 + hat/conf/she.conf | 2 -- project/BasicSettings.scala | 2 +- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala index 0f8c1784f..b783ef97c 100644 --- a/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala +++ b/hat/app/org/hatdex/hat/she/controllers/FunctionManager.scala @@ -131,7 +131,7 @@ class FunctionManager @Inject() ( logger.debug(s"Trigger function $function") functionService.get(function).flatMap { maybeFunction => maybeFunction.map { - case c: FunctionConfiguration => //if c.status.available && c.status.enabled => + case c: FunctionConfiguration if c.status.available && c.status.enabled => functionExecutionDispatcher .trigger(request.dynamicEnvironment.domain, c, useAll)(ec) .map(_ => Ok(Json.toJson(SuccessResponse("Function Executed")))) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index d040cf4cf..cec851efd 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -67,7 +67,7 @@ class FunctionExecutionTriggerHandler @Inject() ( .flatMap { case Some(hatServer) => functionService - .all(active = false)(hatServer.db) // TERRY this must be true in non-debug + .all(active = true)(hatServer.db) // TERRY this must be true in non-debug .map( _.filter({ case FunctionConfiguration(_, @@ -75,7 +75,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(_, true, Some(lastExecution), None) + FunctionStatus(true, true, Some(lastExecution), None) ) if lastExecution.isBefore(DateTime.now().minus(period)) => true case FunctionConfiguration(_, @@ -83,7 +83,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(period), _, - FunctionStatus(_, true, Some(lastExecution), Some(started)) + FunctionStatus(true, true, Some(lastExecution), Some(started)) ) if lastExecution.isBefore(DateTime.now().minus(period)) && started .isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => @@ -93,7 +93,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(_, true, None, None) + FunctionStatus(true, true, None, None) ) => true // no execution recoded yet case FunctionConfiguration(_, @@ -101,7 +101,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerPeriodic(_), _, - FunctionStatus(_, true, None, Some(started)) + FunctionStatus(true, true, None, Some(started)) ) if started.isBefore(DateTime.now().minus(functionExecutionTimeout.toMillis)) => true // no successful execution, current one timed out case FunctionConfiguration(_, @@ -109,7 +109,7 @@ class FunctionExecutionTriggerHandler @Inject() ( _, FunctionTrigger.TriggerIndividual(), _, - FunctionStatus(_, true, _, _) + FunctionStatus(true, true, _, _) ) => true case _ => false // in all other cases, do not trigger diff --git a/hat/conf/base.conf b/hat/conf/base.conf index 2a43cda40..56e26f3f0 100644 --- a/hat/conf/base.conf +++ b/hat/conf/base.conf @@ -1,5 +1,6 @@ akka { loglevel = WARNING + loglevel = ${?AKKA_LOGLEVEL} loggers = ["akka.event.slf4j.Slf4jLogger"] logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" logger-startup-timeout = 30s diff --git a/hat/conf/she.conf b/hat/conf/she.conf index 37b004770..afce723c2 100644 --- a/hat/conf/she.conf +++ b/hat/conf/she.conf @@ -5,8 +5,6 @@ she { region = ${?AWS_REGION} logs = "Tail" mock = false - roleArn = "" - roleArn = ${?SHE_ROLE_ARN} } executionDispatcher { maxHats = 1000 diff --git a/project/BasicSettings.scala b/project/BasicSettings.scala index 89066a67d..7d6e5a35b 100644 --- a/project/BasicSettings.scala +++ b/project/BasicSettings.scala @@ -75,7 +75,7 @@ object BasicSettings extends AutoPlugin { object autoImport { // Do Nothing - // Addet by Terry 20220422 + // Added by Terry 20220422 // Not sure why this block is missing and how you got HAT to compile in the first place? } } From 2b25edc24ac7617aa008e97bad07789c25b906d6 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 02:52:12 +0100 Subject: [PATCH 26/35] Reverted AKKA LOGLEVEL --- .../org/hatdex/hat/she/models/LambdaFunctionExecutable.scala | 2 +- hat/conf/base.conf | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 129113f84..3914fdeb2 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -177,7 +177,7 @@ class AwsLambdaExecutor @Inject() ( val invokeResponse: InvokeResponse = lambdaClient.invoke{request}.get invokeResponse match { case r: InvokeResponse if r.statusCode() == 200 => - logger.debug(s"""Function responded with: + logger.info(s"""Function responded with: | Status: ${r.statusCode()} | Body: ${r.payload().asUtf8String()} | Logs: ${Option(r.logResult()).map(log => java.util.Base64.getDecoder.decode(log))} diff --git a/hat/conf/base.conf b/hat/conf/base.conf index 56e26f3f0..2a43cda40 100644 --- a/hat/conf/base.conf +++ b/hat/conf/base.conf @@ -1,6 +1,5 @@ akka { loglevel = WARNING - loglevel = ${?AKKA_LOGLEVEL} loggers = ["akka.event.slf4j.Slf4jLogger"] logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" logger-startup-timeout = 30s From b46c67cd8fac8301689fdf6d522c2b7ead0445c6 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 03:17:16 +0100 Subject: [PATCH 27/35] Tighter debug --- .../hatdex/hat/she/models/LambdaFunctionExecutable.scala | 3 ++- hat/app/org/hatdex/hat/she/service/FunctionService.scala | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 3914fdeb2..4b3cd8888 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -177,7 +177,7 @@ class AwsLambdaExecutor @Inject() ( val invokeResponse: InvokeResponse = lambdaClient.invoke{request}.get invokeResponse match { case r: InvokeResponse if r.statusCode() == 200 => - logger.info(s"""Function responded with: + logger.debug(s"""Function responded with: | Status: ${r.statusCode()} | Body: ${r.payload().asUtf8String()} | Logs: ${Option(r.logResult()).map(log => java.util.Base64.getDecoder.decode(log))} @@ -187,6 +187,7 @@ class AwsLambdaExecutor @Inject() ( case e => val message = s"Error parsing lambda response: $e" logger.error(message) + logger.error(s"Unable to parse: ${r.payload().asUtf8String()}") throw DataFormatException(message) } Future(jsResponse.get) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionService.scala b/hat/app/org/hatdex/hat/she/service/FunctionService.scala index b19130f69..4be8f338c 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionService.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionService.scala @@ -65,8 +65,10 @@ class FunctionService @Inject() ( val functions = for ((k, v) <- saved ++ registered) - yield k -> (if ((saved contains k) && (registered contains k)) - saved(k).update(v) + yield k -> (if ((saved contains k) && (registered contains k)) { + logger.info(s"Saving $k with $v") + saved(k).update(v) + } else v) functions.values.toSeq From 9f77680cbe7e025dfd6aeb05e6388d1c874940e9 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 04:04:25 +0100 Subject: [PATCH 28/35] Remove howmi --- hat/conf/she.conf | 9 --------- 1 file changed, 9 deletions(-) diff --git a/hat/conf/she.conf b/hat/conf/she.conf index afce723c2..f6945a61c 100644 --- a/hat/conf/she.conf +++ b/hat/conf/she.conf @@ -78,14 +78,5 @@ she { endpoint = "healthsurveyscores" experimental = false } - { - id = "howmi-weizmann-score" - version = "1.0.0" - baseUrl = "howmi-weizmann-score-dev" - baseUrl = ${?DROPS_SHE_BASE_URL} - namespace = "howmi" - endpoint = "healthsurveyscores" - experimental = true - } ] } From 8b4ee22932fd70fc3596c8d85d232ce149e365d4 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 04:35:06 +0100 Subject: [PATCH 29/35] Handle SHE function invokation response error --- .../she/models/LambdaFunctionExecutable.scala | 45 ++++++++++--------- .../hat/she/service/FunctionService.scala | 1 - hat/conf/she.conf | 9 ++++ 3 files changed, 34 insertions(+), 21 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 4b3cd8888..71591b9e5 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -176,26 +176,31 @@ class AwsLambdaExecutor @Inject() ( else { val invokeResponse: InvokeResponse = lambdaClient.invoke{request}.get invokeResponse match { - case r: InvokeResponse if r.statusCode() == 200 => - logger.debug(s"""Function responded with: - | Status: ${r.statusCode()} - | Body: ${r.payload().asUtf8String()} - | Logs: ${Option(r.logResult()).map(log => java.util.Base64.getDecoder.decode(log))} - """.stripMargin) - val jsResponse = - Json.parse(r.payload().asUtf8String()).validate[T] recover { - case e => - val message = s"Error parsing lambda response: $e" - logger.error(message) - logger.error(s"Unable to parse: ${r.payload().asUtf8String()}") - throw DataFormatException(message) - } - Future(jsResponse.get) - case r => - val message = - s"Retrieving SHE function configuration failed: $r, ${r.payload().asUtf8String()}" - logger.error(message) - throw new ApiException(message) + case r: InvokeResponse if r.functionError() == null => + logger.debug(s"""Function responded with: + | Status: ${r.statusCode()} + | Body: ${r.payload().asUtf8String()} + | Logs: ${Option(r.logResult()).map(log => java.util.Base64.getDecoder.decode(log))} + """.stripMargin) + val jsResponse = + Json.parse(r.payload().asUtf8String()).validate[T] recover { + case e => + val message = s"Error parsing lambda response: $e" + logger.error(message) + logger.error(s"Unable to parse: ${r.payload().asUtf8String()}") + throw DataFormatException(message) + } + Future(jsResponse.get) + case r: InvokeResponse if r.functionError() != null => + val message = + s"Retrieving SHE function Response Error: ${r.functionError()}" + logger.error(message) + throw new ApiException(message) + case r => + val message = + s"Retrieving SHE function Response FAILED: $r, ${r.payload().asUtf8String()}" + logger.error(message) + throw new ApiException(message) } } diff --git a/hat/app/org/hatdex/hat/she/service/FunctionService.scala b/hat/app/org/hatdex/hat/she/service/FunctionService.scala index 4be8f338c..01eab6e1d 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionService.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionService.scala @@ -66,7 +66,6 @@ class FunctionService @Inject() ( val functions = for ((k, v) <- saved ++ registered) yield k -> (if ((saved contains k) && (registered contains k)) { - logger.info(s"Saving $k with $v") saved(k).update(v) } else v) diff --git a/hat/conf/she.conf b/hat/conf/she.conf index f6945a61c..afce723c2 100644 --- a/hat/conf/she.conf +++ b/hat/conf/she.conf @@ -78,5 +78,14 @@ she { endpoint = "healthsurveyscores" experimental = false } + { + id = "howmi-weizmann-score" + version = "1.0.0" + baseUrl = "howmi-weizmann-score-dev" + baseUrl = ${?DROPS_SHE_BASE_URL} + namespace = "howmi" + endpoint = "healthsurveyscores" + experimental = true + } ] } From dfd4646122600b1a24f24dc24bef0a52ce3bef59 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Sun, 24 Apr 2022 13:19:42 +0100 Subject: [PATCH 30/35] Switch to DefaultCredentialsProvider since WebIdentityTokenFileCredentialsProvider is now in default chain --- .../org/hatdex/hat/she/models/LambdaFunctionExecutable.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index 71591b9e5..eb84ec07e 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -37,7 +37,7 @@ import software.amazon.awssdk.core.SdkBytes import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.lambda.LambdaAsyncClient import software.amazon.awssdk.services.lambda.model.{ InvokeRequest, InvokeResponse } -import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider import javax.inject.Inject import scala.concurrent.{ ExecutionContext, Future } @@ -164,7 +164,7 @@ class AwsLambdaExecutor @Inject() ( LambdaAsyncClient .builder() .region(Region.of(configuration.get[String]("she.aws.region"))) - .credentialsProvider(WebIdentityTokenFileCredentialsProvider.create()) + .credentialsProvider(DefaultCredentialsProvider.create()) .build() actorSystem.registerOnTermination(lambdaClient.close()) From 5bdb0dc98dd1c4f48dc648c78eb28fda1ae0960c Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Mon, 25 Apr 2022 06:38:52 +0100 Subject: [PATCH 31/35] Remove redundant comment --- .../hatdex/hat/she/service/FunctionExecutionDispatcher.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index cec851efd..fd4734fb0 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -67,7 +67,7 @@ class FunctionExecutionTriggerHandler @Inject() ( .flatMap { case Some(hatServer) => functionService - .all(active = true)(hatServer.db) // TERRY this must be true in non-debug + .all(active = true)(hatServer.db) .map( _.filter({ case FunctionConfiguration(_, From 8d698ce288e602eabbb2f641d712b82f6d33efa7 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Mon, 25 Apr 2022 10:10:10 +0100 Subject: [PATCH 32/35] Updated version for PR --- project/Dependencies.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 88f9cc242..5adaed5e1 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -53,7 +53,7 @@ object Dependencies { } private object Version { - val AlpakkaAwsLambda = "3.0.4" + val AlpakkaAwsLambda = "1.1.2" val CirceConfig = "0.8.0" val PrettyTime = "5.0.2.Final" val ScalaTestplusMockito = "3.2.10.0" From cdb9acf9894f6704c2630546cfd380791159937a Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Mon, 25 Apr 2022 11:46:12 +0100 Subject: [PATCH 33/35] Update FunctionExecutionDispatcher.scala --- .../hatdex/hat/she/service/FunctionExecutionDispatcher.scala | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index fd4734fb0..98d090d9d 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -202,10 +202,7 @@ class FunctionExecutionTriggerHandler @Inject() ( functionService .run(conf, conf.status.lastExecution, useAll)(hatServer) .recover { - case e => { - logger.error(e.getMessage(), e) - throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) - } + case e => throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) } } getOrElse { Future.failed( From d306bca91b8f9bc979517efe07f16d0fa94baa89 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Mon, 25 Apr 2022 11:52:01 +0100 Subject: [PATCH 34/35] Revert formatting --- .../hatdex/hat/she/service/FunctionExecutionDispatcher.scala | 2 +- hat/app/org/hatdex/hat/she/service/FunctionService.scala | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala index 98d090d9d..27b5fa18d 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionExecutionDispatcher.scala @@ -202,7 +202,7 @@ class FunctionExecutionTriggerHandler @Inject() ( functionService .run(conf, conf.status.lastExecution, useAll)(hatServer) .recover { - case e => throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) + case e => throw SHEFunctionExecutionFailureException(s"$hat function ${conf.id} failed", e) } } getOrElse { Future.failed( diff --git a/hat/app/org/hatdex/hat/she/service/FunctionService.scala b/hat/app/org/hatdex/hat/she/service/FunctionService.scala index 01eab6e1d..b19130f69 100644 --- a/hat/app/org/hatdex/hat/she/service/FunctionService.scala +++ b/hat/app/org/hatdex/hat/she/service/FunctionService.scala @@ -65,9 +65,8 @@ class FunctionService @Inject() ( val functions = for ((k, v) <- saved ++ registered) - yield k -> (if ((saved contains k) && (registered contains k)) { - saved(k).update(v) - } + yield k -> (if ((saved contains k) && (registered contains k)) + saved(k).update(v) else v) functions.values.toSeq From 85ed9499629eb73996b988d8f5037b6668fec695 Mon Sep 17 00:00:00 2001 From: Terry Lee Date: Mon, 25 Apr 2022 12:44:06 +0100 Subject: [PATCH 35/35] Scala syntax update --- .../org/hatdex/hat/she/models/LambdaFunctionExecutable.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala index eb84ec07e..3f79bbc07 100644 --- a/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala +++ b/hat/app/org/hatdex/hat/she/models/LambdaFunctionExecutable.scala @@ -174,8 +174,7 @@ class AwsLambdaExecutor @Inject() ( )(implicit jsonFormatter: Format[T]): Future[T] = if (mock) Future.successful(null.asInstanceOf[T]) else { - val invokeResponse: InvokeResponse = lambdaClient.invoke{request}.get - invokeResponse match { + lambdaClient.invoke{request}.get match { case r: InvokeResponse if r.functionError() == null => logger.debug(s"""Function responded with: | Status: ${r.statusCode()}