From 171214eea067fa0b75c3ba971c6545d17b5265fa Mon Sep 17 00:00:00 2001 From: Roberto Tyley <52038+rtyley@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:37:10 +0000 Subject: [PATCH] Enable ETag caching with Facia API requests With https://github.com/guardian/facia-scala-client/pull/287, the Facia client has been updated so that it can take advantage of ETag-caching, using the https://github.com/guardian/etag-caching library already used in Frontend since August 2023 with https://github.com/guardian/frontend/pull/26338. As some changes were necessary to the `etag-caching` library to facilitate this, we need to update the version of `etag-caching` used by Frontend to v7.0.0. This does involve switching to AWS SDK v2 - not everywhere in Frontend, just for where we use the Facia client! --- build.sbt | 3 +- common/app/common/configuration.scala | 42 ++++++++++------------ common/app/services/fronts/FrontsApi.scala | 20 ++++++----- common/app/utils/AWSv2.scala | 21 +++++++++++ project/Dependencies.scala | 5 +-- 5 files changed, 56 insertions(+), 35 deletions(-) diff --git a/build.sbt b/build.sbt index fea5bd9fe35a..28963fcd10fa 100644 --- a/build.sbt +++ b/build.sbt @@ -25,7 +25,8 @@ val common = library("common") awsKinesis, awsS3, awsSns, - awsSts, + awsSts, // AWS SDK v1 still used for CAPI-preview related code for now + awsV2Sts, // AWS SDK v2 used for Fronts API access awsSqs, awsSsm, eTagCachingS3, diff --git a/common/app/common/configuration.scala b/common/app/common/configuration.scala index 228ed5a4b9e5..50af3e0abe4c 100644 --- a/common/app/common/configuration.scala +++ b/common/app/common/configuration.scala @@ -1,9 +1,5 @@ package common -import java.io.{File, FileInputStream} -import java.nio.charset.Charset -import java.util.Map.Entry - import com.amazonaws.AmazonClientException import com.amazonaws.auth._ import com.amazonaws.auth.profile.ProfileCredentialsProvider @@ -12,9 +8,15 @@ import common.Environment.{app, awsRegion, stage} import conf.{Configuration, Static} import org.apache.commons.io.IOUtils import services.ParameterStore +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.core.exception.SdkClientException +import utils.AWSv2 -import scala.jdk.CollectionConverters._ +import java.io.{File, FileInputStream} +import java.nio.charset.Charset +import java.util.Map.Entry import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} class BadConfigurationException(msg: String) extends RuntimeException(msg) @@ -231,14 +233,10 @@ class GuardianConfiguration extends GuLogging { lazy val capiPreviewRoleToAssume: Option[String] = configuration.getStringProperty("content.api.preview.roleToAssume") - lazy val capiPreviewCredentials: AWSCredentialsProvider = { - new AWSCredentialsProviderChain( - List( - Some(new ProfileCredentialsProvider("capi")), - capiPreviewRoleToAssume.map(new STSAssumeRoleSessionCredentialsProvider.Builder(_, "capi").build()), - ).flatten: _*, - ) - } + lazy val capiPreviewCredentials: AWSCredentialsProvider = new AWSCredentialsProviderChain( + Seq(new ProfileCredentialsProvider("capi")) ++ + capiPreviewRoleToAssume.map(new STSAssumeRoleSessionCredentialsProvider.Builder(_, "capi").build()): _*, + ) lazy val nextPreviousPageSize: Int = configuration.getIntegerProperty("content.api.nextPreviousPageSize").getOrElse(50) @@ -597,24 +595,20 @@ class GuardianConfiguration extends GuLogging { lazy val stsRoleToAssume = configuration.getStringProperty("aws.cmsFronts.account.role") - def crossAccountMandatoryCredentials: AWSCredentialsProvider = + def crossAccountMandatoryCredentials: AwsCredentialsProvider = crossAccountCredentials.getOrElse( throw new BadConfigurationException("AWS credentials for cross account are not configured"), ) - lazy val crossAccountCredentials: Option[AWSCredentialsProvider] = faciatool.stsRoleToAssume.flatMap { role => - val provider = new AWSCredentialsProviderChain( - new ProfileCredentialsProvider("cmsFronts"), - new STSAssumeRoleSessionCredentialsProvider.Builder(role, "frontend").build(), - ) + lazy val crossAccountCredentials: Option[AwsCredentialsProvider] = faciatool.stsRoleToAssume.map { roleArn => + val provider = AWSv2.stsCredentialsForDevAndProd("cmsFronts", roleArn) - // this is a bit of a convoluted way to check whether we actually have credentials. - // I guess in an ideal world there would be some sort of isConfigued() method... + // Check whether we actually have credentials! try { - provider.getCredentials - Some(provider) + provider.resolveCredentials() + provider } catch { - case ex: AmazonClientException => + case ex: SdkClientException => log.error("amazon client cross account exception", ex) throw ex } diff --git a/common/app/services/fronts/FrontsApi.scala b/common/app/services/fronts/FrontsApi.scala index 408bef04e4e1..0d3604c1427d 100644 --- a/common/app/services/fronts/FrontsApi.scala +++ b/common/app/services/fronts/FrontsApi.scala @@ -1,21 +1,25 @@ package services.fronts -import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client} -import com.gu.facia.client.{AmazonSdkS3Client, ApiClient} +import com.gu.etagcaching.aws.sdkv2.s3.S3ObjectFetching +import com.gu.facia.client.{ApiClient, Environment} import conf.Configuration +import software.amazon.awssdk.services.s3.S3AsyncClient +import utils.AWSv2 + import scala.concurrent.ExecutionContext object FrontsApi { def crossAccountClient(implicit ec: ExecutionContext): ApiClient = { - val client: AmazonS3 = AmazonS3Client.builder - .withCredentials(Configuration.faciatool.crossAccountMandatoryCredentials) - .withRegion(conf.Configuration.aws.region) + val amazonS3Client = S3AsyncClient.builder + .credentialsProvider(Configuration.faciatool.crossAccountMandatoryCredentials) + .region(AWSv2.region) .build() - ApiClient( + + ApiClient.withCaching( Configuration.faciatool.crossAccountSourceBucket, - Configuration.facia.stage.toUpperCase, - AmazonSdkS3Client(client), + Environment(Configuration.facia.stage.toUpperCase), + S3ObjectFetching.byteArraysWith(amazonS3Client), ) } } diff --git a/common/app/utils/AWSv2.scala b/common/app/utils/AWSv2.scala index 6f45d7a291a3..8895e518e16c 100644 --- a/common/app/utils/AWSv2.scala +++ b/common/app/utils/AWSv2.scala @@ -6,6 +6,9 @@ import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient import software.amazon.awssdk.regions.Region import software.amazon.awssdk.regions.Region.EU_WEST_1 import software.amazon.awssdk.services.s3.{S3AsyncClient, S3AsyncClientBuilder} +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest +import software.amazon.awssdk.services.sts.{StsClient, StsClientBuilder} object AWSv2 { val region: Region = EU_WEST_1 @@ -22,4 +25,22 @@ object AWSv2 { val S3Async: S3AsyncClient = build[S3AsyncClient, S3AsyncClientBuilder]( S3AsyncClient.builder().httpClientBuilder(NettyNioAsyncHttpClient.builder().maxConcurrency(250)), ) + + val STS: StsClient = build[StsClient, StsClientBuilder](StsClient.builder()) + + def stsCredentialsForDevAndProd(devProfile: String, roleArn: String): AwsCredentialsProviderChain = + credentialsForDevAndProd( + devProfile, + StsAssumeRoleCredentialsProvider + .builder() + .stsClient(STS) + .refreshRequest( + AssumeRoleRequest.builder + .roleSessionName("frontend") + .roleArn(roleArn) + .build, + ) + .build(), + ) + } diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 7ee8b1c9af7c..d239252be8c4 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -7,7 +7,7 @@ object Dependencies { val awsVersion = "1.12.758" val awsSdk2Version = "2.26.27" val capiVersion = "33.0.0" - val faciaVersion = "14.0.0" + val faciaVersion = "14.0.1" val dispatchVersion = "0.13.1" val romeVersion = "1.0" val jerseyVersion = "1.19.4" @@ -19,10 +19,11 @@ object Dependencies { val awsEc2 = "com.amazonaws" % "aws-java-sdk-ec2" % awsVersion val awsKinesis = "com.amazonaws" % "aws-java-sdk-kinesis" % awsVersion val awsS3 = "com.amazonaws" % "aws-java-sdk-s3" % awsVersion - val eTagCachingS3 = "com.gu.etag-caching" %% "aws-s3-sdk-v2" % "4.0.1" + val eTagCachingS3 = "com.gu.etag-caching" %% "aws-s3-sdk-v2" % "7.0.0" val awsSes = "com.amazonaws" % "aws-java-sdk-ses" % awsVersion val awsSns = "com.amazonaws" % "aws-java-sdk-sns" % awsVersion val awsSts = "com.amazonaws" % "aws-java-sdk-sts" % awsVersion + val awsV2Sts = "software.amazon.awssdk" % "sts" % awsSdk2Version val awsSqs = "com.amazonaws" % "aws-java-sdk-sqs" % awsVersion val awsSsm = "com.amazonaws" % "aws-java-sdk-ssm" % awsVersion val awsElasticloadbalancing = "com.amazonaws" % "aws-java-sdk-elasticloadbalancing" % awsVersion