-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add ETag-caching (and AWS SDK v2) support
This change adds these improvements: * Facia data is only re-downloaded & re-parsed if the S3 content has _changed_, thanks to ETag-caching - see https://github.com/guardian/etag-caching . This library has already been used in DotCom PROD with guardian/frontend#26338 * AWS SDK v2: the FAPI client itself now has a `fapi-s3-sdk-v2` artifact. An example PR consuming this updated version of the FAPI client is at: guardian/ophan#5506 Updated FAPI artifact layout ---------------------------- To use FAPI with the new AWS SDK v2 support, users must now have a dependency on *two* FAPI artifacts: * `fapi-s3-sdk-v2` * `fapi-client-playXX` Due to needing to support the matrix of: * AWS SDK v1 & v2 * Play-JSON 2.8, 2.9, 3.0 ...it's best not to try to produce an artifact that corresponds to every single combination of those! Consequently, we provide an artifacts that are specific to the different versions of AWS SDK (or at least, could do - if AWS SDK v1 was moved out of common code), and artifacts that are specific to the different versions of Play-JSON, and allow the user to combine them as needed. A similar approach was used with `guardian/play-secret-rotation`: guardian/play-secret-rotation#8 In order for the different artifacts to have interfaces they can use to join together and become a single useful Facia client, we have a `fapi-client-core` artifact. Any code that doesn't depend on the JSON classes, or the actual AWS SDK version (which isn't much!), can live in there. In particular, we have: * `com.gu.facia.client.ApiClient`, an existing type that is now a trait, with 2 implementations - one that uses the existing `com.gu.facia.client.S3Client` abstraction on S3 behaviour * `com.gu.facia.client.etagcaching.fetching.S3FetchBehaviour`, a new trait that exposes just enough interface to allow the conditional fetching used for ETag-based caching, but doesn't tie you to any specific version of the AWS SDK.
- Loading branch information
Showing
11 changed files
with
194 additions
and
65 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 4 additions & 8 deletions
12
.../scala/com/gu/facia/client/S3Client.scala → ...m/gu/facia/client/AmazonSdkS3Client.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
100 changes: 77 additions & 23 deletions
100
facia-json/src/main/scala/com/gu/facia/client/ApiClient.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,37 +1,91 @@ | ||
| package com.gu.facia.client | ||
|
|
||
| import com.gu.facia.client.models.{ConfigJson, CollectionJson} | ||
| import com.gu.etagcaching.FreshnessPolicy.AlwaysWaitForRefreshedValue | ||
| import com.gu.etagcaching.aws.s3.ObjectId | ||
| import com.gu.etagcaching.{ConfigCache, ETagCache} | ||
| import com.gu.facia.client.etagcaching.fetching.S3FetchBehaviour | ||
| import com.gu.facia.client.models.{CollectionJson, ConfigJson} | ||
| import play.api.libs.json.{Format, Json} | ||
|
|
||
| import scala.concurrent.{ExecutionContext, Future} | ||
|
|
||
| object ApiClient { | ||
| val Encoding = "utf-8" | ||
| trait ApiClient { | ||
| def config: Future[ConfigJson] | ||
|
|
||
| def collection(id: String): Future[Option[CollectionJson]] | ||
| } | ||
|
|
||
| case class ApiClient( | ||
| object ApiClient { | ||
| private val Encoding = "utf-8" | ||
|
|
||
| def apply( | ||
| bucket: String, | ||
| /** e.g., CODE, PROD, DEV ... */ | ||
| environment: String, | ||
| environment: String, // e.g., CODE, PROD, DEV ... | ||
| s3Client: S3Client | ||
| )(implicit executionContext: ExecutionContext) { | ||
| import com.gu.facia.client.ApiClient._ | ||
|
|
||
| private def retrieve[A: Format](key: String): Future[Option[A]] = s3Client.get(bucket, key) map { | ||
| case FaciaSuccess(bytes) => | ||
| Some(Json.fromJson[A](Json.parse(new String(bytes, Encoding))) getOrElse { | ||
| throw new JsonDeserialisationError(s"Could not deserialize JSON in $bucket, $key") | ||
| }) | ||
| case FaciaNotAuthorized(message) => throw new BackendError(message) | ||
| case FaciaNotFound(_) => None | ||
| case FaciaUnknownError(message) => throw new BackendError(message) | ||
| )(implicit executionContext: ExecutionContext): ApiClient = new ApiClient { | ||
| val env: Environment = Environment(environment) | ||
|
|
||
| private def retrieve[A: Format](key: String): Future[Option[A]] = s3Client.get(bucket, key).map(translateFaciaResult[A](_)) | ||
|
|
||
| def config: Future[ConfigJson] = | ||
| retrieve[ConfigJson](env.configS3Path).map(getOrWarnAboutMissingConfig) | ||
|
|
||
| def collection(id: String): Future[Option[CollectionJson]] = | ||
| retrieve[CollectionJson](env.collectionS3Path(id)) | ||
| } | ||
|
|
||
| private def getOrWarnAboutMissingConfig(configOpt: Option[ConfigJson]): ConfigJson = | ||
| configOpt getOrElse throwConfigMissingException() | ||
|
|
||
| private def throwConfigMissingException() = throw BackendError("Config was missing!! NOT GOOD!") | ||
|
|
||
| private def translateFaciaResult[B: Format](faciaResult: FaciaResult): Option[B] = faciaResult match { | ||
| case FaciaSuccess(bytes) => Some(parseBytes(bytes)) | ||
| case FaciaNotAuthorized(message) => throw BackendError(message) | ||
| case FaciaNotFound(_) => None | ||
| case FaciaUnknownError(message) => throw BackendError(message) | ||
| } | ||
|
|
||
| def config: Future[ConfigJson] = | ||
| retrieve[ConfigJson](s"$environment/frontsapi/config/config.json").map(_ getOrElse { | ||
| throw new BackendError("Config was missing!! OH MY GOD") | ||
| }) | ||
| private def parseBytes[B: Format](bytes: Array[Byte]): B = | ||
| Json.fromJson[B](Json.parse(new String(bytes, Encoding))) getOrElse { | ||
| throw JsonDeserialisationError(s"Could not deserialize JSON") | ||
| } | ||
|
|
||
| def withCaching( | ||
| bucket: String, | ||
| s3FetchBehaviour: S3FetchBehaviour, | ||
| environment: Environment, | ||
| configureCollectionCache: ConfigCache = _.maximumSize(10000) // at most 1GB RAM worst case | ||
| )(implicit ec: ExecutionContext): ApiClient = | ||
| new ApiClient { | ||
| private val fetching = | ||
| s3FetchBehaviour.fetching.keyOn[String](path => ObjectId(bucket, path)) | ||
| /* TODO - .suppressExceptionLoggingIf(s3FetchBehaviour.fetchExceptionIndicatesContentMissing) | ||
| * Need to settle on an approach - see https://github.com/guardian/etag-caching/pull/32 | ||
| */ | ||
|
|
||
| def eTagCache[B: Format](configureCache: ConfigCache) = new ETagCache( | ||
| fetching.thenParsing(parseBytes[B]), | ||
| AlwaysWaitForRefreshedValue, | ||
| configureCache | ||
| ) | ||
|
|
||
| private val configCache = eTagCache[ConfigJson](_.maximumSize(1)) | ||
| private val collectionCache = eTagCache[CollectionJson](configureCollectionCache) | ||
|
|
||
| override def config: Future[ConfigJson] = configCache.get(environment.configS3Path).recover { | ||
| case t if s3FetchBehaviour.fetchExceptionIndicatesContentMissing(t) => | ||
| throwConfigMissingException() | ||
| } | ||
|
|
||
| def collection(id: String): Future[Option[CollectionJson]] = | ||
| retrieve[CollectionJson](s"$environment/frontsapi/collection/$id/collection.json") | ||
| override def collection(id: String): Future[Option[CollectionJson]] = | ||
| collectionCache.get(environment.collectionS3Path(id)).map(Some(_)).recover { | ||
| case t if { | ||
| val boo = s3FetchBehaviour.fetchExceptionIndicatesContentMissing(t) | ||
| println(s"Exception is content missing: $boo") | ||
| boo | ||
| } => None | ||
| case t => throw BackendError(t.getMessage) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
15 changes: 15 additions & 0 deletions
15
fapi-client-core/src/main/scala/com/gu/facia/client/Environment.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| package com.gu.facia.client | ||
|
|
||
| case class Environment(name: String) { | ||
| private val s3PathPrefix: String = s"$name/frontsapi" | ||
|
|
||
| val configS3Path: String = s"$s3PathPrefix/config/config.json" | ||
| def collectionS3Path(id: String): String = s"$s3PathPrefix/collection/$id/collection.json" | ||
| } | ||
|
|
||
| object Environment { | ||
| val Prod = Environment("PROD") | ||
| val Code = Environment("CODE") | ||
| val Dev = Environment("DEV") | ||
| val Test = Environment("TEST") | ||
| } |
File renamed without changes.
File renamed without changes.
10 changes: 10 additions & 0 deletions
10
fapi-client-core/src/main/scala/com/gu/facia/client/S3Client.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package com.gu.facia.client | ||
|
|
||
| import scala.concurrent.Future | ||
|
|
||
| /** For mocking in tests, but also to allow someone to define a properly asynchronous S3 client. (The one in the AWS | ||
| * SDK is unfortunately synchronous only.) | ||
| */ | ||
| trait S3Client { | ||
| def get(bucket: String, path: String): Future[FaciaResult] | ||
| } |
10 changes: 10 additions & 0 deletions
10
...lient-core/src/main/scala/com/gu/facia/client/etagcaching/fetching/S3FetchBehaviour.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| package com.gu.facia.client.etagcaching.fetching | ||
|
|
||
| import com.gu.etagcaching.aws.s3.ObjectId | ||
| import com.gu.etagcaching.fetching.Fetching | ||
|
|
||
| trait S3FetchBehaviour { | ||
| val fetching: Fetching[ObjectId, Array[Byte]] | ||
|
|
||
| def fetchExceptionIndicatesContentMissing(t: Throwable): Boolean | ||
| } |
19 changes: 19 additions & 0 deletions
19
fapi-s3-sdk-v2/src/main/scala/com/gu/facia/client/aws/sdkv2/s3/AwsSdkV2.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package com.gu.facia.client.aws.sdkv2.s3 | ||
|
|
||
| import com.gu.etagcaching.aws.s3.ObjectId | ||
| import com.gu.etagcaching.aws.sdkv2.s3.S3ObjectFetching | ||
| import com.gu.etagcaching.aws.sdkv2.s3.response.Transformer.Bytes | ||
| import com.gu.etagcaching.fetching.Fetching | ||
| import com.gu.facia.client.etagcaching.fetching.S3FetchBehaviour | ||
| import software.amazon.awssdk.services.s3.S3AsyncClient | ||
| import software.amazon.awssdk.services.s3.model.NoSuchKeyException | ||
|
|
||
| case class AwsSdkV2(s3AsyncClient: S3AsyncClient) extends S3FetchBehaviour { | ||
| override val fetching: Fetching[ObjectId, Array[Byte]] = | ||
| S3ObjectFetching(s3AsyncClient, Bytes).mapResponse(_.asByteArray()) | ||
|
|
||
| override def fetchExceptionIndicatesContentMissing(t: Throwable): Boolean = t match { | ||
| case _: NoSuchKeyException => true | ||
| case _ => false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters