diff --git a/common/app/common/metrics.scala b/common/app/common/metrics.scala index efeb4269984b..45b93bd8e847 100644 --- a/common/app/common/metrics.scala +++ b/common/app/common/metrics.scala @@ -138,6 +138,7 @@ object FaciaPressMetrics { val AuPressLatencyMetric = DurationMetric("au-press-latency", StandardUnit.Milliseconds) val AllFrontsPressLatencyMetric = DurationMetric("front-press-latency", StandardUnit.Milliseconds) val FrontPressContentSize = SamplerMetric("front-press-content-size", StandardUnit.Bytes) + val FrontPressContentSizeLite = SamplerMetric("front-press-content-size-lite", StandardUnit.Bytes) val FrontDecodingLatency = DurationMetric("front-decoding-latency", StandardUnit.Milliseconds) } diff --git a/common/app/layout/Front.scala b/common/app/layout/Front.scala index 4eb9b1067b9a..fa7f84b9408f 100644 --- a/common/app/layout/Front.scala +++ b/common/app/layout/Front.scala @@ -297,7 +297,10 @@ object Front extends implicits.Collections { type TrailUrl = String def itemsVisible(containerDefinition: ContainerDefinition): Int = - containerDefinition.slices.flatMap(_.layout.columns.map(_.numItems)).sum + itemsVisible(containerDefinition.slices) + + def itemsVisible(slices: Seq[Slice]): Int = + slices.flatMap(_.layout.columns.map(_.numItems)).sum // Never de-duplicate snaps. def participatesInDeduplication(faciaContent: PressedContent): Boolean = faciaContent.properties.embedType.isEmpty diff --git a/common/app/layout/slices/Container.scala b/common/app/layout/slices/Container.scala index 7b7a42ffdf41..e081bd3e49db 100644 --- a/common/app/layout/slices/Container.scala +++ b/common/app/layout/slices/Container.scala @@ -1,7 +1,8 @@ package layout.slices -import model.pressed.CollectionConfig +import model.pressed.{CollectionConfig, PressedContent} import common.Logging +import layout.Front import model.facia.PressedCollection import scala.collection.immutable.Iterable @@ -31,6 +32,16 @@ object Container extends Logging { def fromConfig(collectionConfig: CollectionConfig): Container = resolve(collectionConfig.collectionType) + def storiesCount(collectionConfig: CollectionConfig, items: Seq[PressedContent]): Option[Int] = { + resolve(collectionConfig.collectionType) match { + case Dynamic(dynamicPackage) => dynamicPackage + .slicesFor(items.map(Story.fromFaciaContent)) + .map(Front.itemsVisible) + case Fixed(fixedContainer) => Some(Front.itemsVisible(fixedContainer.slices)) + case _ => None + } + } + def fromPressedCollection(pressedCollection: PressedCollection, omitMPU: Boolean, adFree: Boolean): Container = { val container = resolve(pressedCollection.collectionType) container match { diff --git a/common/app/model/PressedPage.scala b/common/app/model/PressedPage.scala index 388baade3bfe..7e4e600b9f3e 100644 --- a/common/app/model/PressedPage.scala +++ b/common/app/model/PressedPage.scala @@ -60,6 +60,34 @@ object PressedPage { } } +sealed trait PressedPageType { + def suffix: String +} + +case object FullType extends PressedPageType { + override def suffix = "" +} + +case object LiteType extends PressedPageType { + override def suffix = ".lite" +} + +case class PressedPageVersions(lite: PressedPage, full: PressedPage) + +object PressedPageVersions { + def fromPressedCollections(id: String, + seoData: SeoData, + frontProperties: FrontProperties, + pressedCollections: List[PressedCollectionVersions]): PressedPageVersions = { + PressedPageVersions( + PressedPage(id, seoData, frontProperties, pressedCollections.map(_.lite)), + PressedPage(id, seoData, frontProperties, pressedCollections.map(_.full)) + ) + } +} + +case class PressedCollectionVersions(lite: PressedCollection, full: PressedCollection) + case class PressedPage ( id: String, seoData: SeoData, diff --git a/common/app/services/S3.scala b/common/app/services/S3.scala index e0e33cd3ef12..0df373495d73 100644 --- a/common/app/services/S3.scala +++ b/common/app/services/S3.scala @@ -12,6 +12,7 @@ import com.amazonaws.services.s3.model._ import com.amazonaws.util.StringInputStream import common.Logging import conf.Configuration +import model.PressedPageType import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.ws.{WSClient, WSRequest} import sun.misc.BASE64Encoder @@ -153,19 +154,12 @@ object S3FrontsApi extends S3 { lazy val stage = Configuration.facia.stage.toUpperCase val namespace = "frontsapi" lazy val location = s"$stage/$namespace" - private val filename = "pressed.v2.json" - def getLiveFapiPressedKeyForPath(path: String): String = - s"$location/pressed/live/$path/fapi/$filename" + private def putFapiPressedJson(live: String, path: String, json: String, suffix: String): Unit = + putPrivateGzipped(s"$location/pressed/$live/$path/fapi/pressed.v2$suffix.json", json, "application/json") - def getDraftFapiPressedKeyForPath(path: String): String = - s"$location/pressed/draft/$path/fapi/$filename" - - def putLiveFapiPressedJson(path: String, json: String): Unit = - putPrivateGzipped(getLiveFapiPressedKeyForPath(path), json, "application/json") - - def putDraftFapiPressedJson(path: String, json: String): Unit = - putPrivateGzipped(getDraftFapiPressedKeyForPath(path), json, "application/json") + def putLiveFapiPressedJson(path: String, json: String, pressedType: PressedPageType): Unit = putFapiPressedJson("live", path, json, pressedType.suffix) + def putDraftFapiPressedJson(path: String, json: String, pressedType: PressedPageType): Unit = putFapiPressedJson("draft", path, json, pressedType.suffix) } object S3Archive extends S3 { diff --git a/facia-press/app/controllers/Application.scala b/facia-press/app/controllers/Application.scala index ca36c515b186..6a992a7a9bb4 100644 --- a/facia-press/app/controllers/Application.scala +++ b/facia-press/app/controllers/Application.scala @@ -1,10 +1,11 @@ package controllers +import com.gu.facia.api.Response import common.ImplicitControllerExecutionContext import conf.Configuration import conf.switches.Switches.FaciaPressOnDemand import frontpress.{DraftFapiFrontPress, LiveFapiFrontPress} -import model.NoCache +import model.{NoCache, PressedPage} import play.api.libs.json.Json import play.api.mvc._ import services.ConfigAgent @@ -21,16 +22,26 @@ class Application(liveFapiFrontPress: LiveFapiFrontPress, draftFapiFrontPress: D NoCache(Ok(ConfigAgent.contentsAsJsonString).withHeaders("Content-Type" -> "application/json")) } - def generateLivePressedFor(path: String): Action[AnyContent] = Action.async { request => - liveFapiFrontPress.getPressedFrontForPath(path) - .map(Json.toJson(_)) + def generateLivePressedFor(path: String): Action[AnyContent] = Action.async { _ => + val front = liveFapiFrontPress.getPressedFrontForPath(path).map(_.full) + frontToResult(front) + } + + def generateLiteLivePressedFor(path: String): Action[AnyContent] = Action.async { _ => + val front = liveFapiFrontPress.getPressedFrontForPath(path).map(_.lite) + frontToResult(front) + } + + private def frontToResult(front: Response[PressedPage]): Future[Result] = { + front.map(Json.toJson(_)) .map(Json.prettyPrint) .map(Ok.apply(_)) .map(NoCache.apply) .fold( apiError => InternalServerError(apiError.message), successJson => successJson.withHeaders("Content-Type" -> "application/json") - )} + ) + } private def handlePressRequest(path: String, liveOrDraft: String)(f: (String) => Future[_]): Future[Result] = if (FaciaPressOnDemand.isSwitchedOn) { diff --git a/facia-press/app/frontpress/FapiFrontPress.scala b/facia-press/app/frontpress/FapiFrontPress.scala index 4e2176c50bd1..cdad2c119598 100644 --- a/facia-press/app/frontpress/FapiFrontPress.scala +++ b/facia-press/app/frontpress/FapiFrontPress.scala @@ -1,5 +1,6 @@ package frontpress +import metrics.SamplerMetric import com.gu.contentapi.client.ContentApiClientLogic import com.gu.contentapi.client.model.v1.ItemResponse import com.gu.contentapi.client.model.{ItemQuery, SearchQuery} @@ -14,7 +15,7 @@ import conf.Configuration import conf.switches.Switches.FaciaInlineEmbeds import contentapi.{CapiHttpClient, CircuitBreakingContentApiClient, ContentApiClient, QueryDefaults} import services.fronts.FrontsApi -import model._ +import model.{PressedPage, _} import model.facia.PressedCollection import model.pressed._ import org.joda.time.DateTime @@ -22,12 +23,14 @@ import play.api.libs.json._ import play.api.libs.ws.WSClient import services.{ConfigAgent, S3FrontsApi} import implicits.Booleans._ +import layout.slices.Container + import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} class LiveFapiFrontPress(val wsClient: WSClient, val capiClientForFrontsSeo: ContentApiClient)(implicit ec: ExecutionContext) extends FapiFrontPress { - override def putPressedJson(path: String, json: String): Unit = S3FrontsApi.putLiveFapiPressedJson(path, json) + override def putPressedJson(path: String, json: String, pressedType: PressedPageType): Unit = S3FrontsApi.putLiveFapiPressedJson(path, json, pressedType) override def isLiveContent: Boolean = true override implicit val capiClient: ContentApiClientLogic = CircuitBreakingContentApiClient( @@ -55,7 +58,7 @@ class DraftFapiFrontPress(val wsClient: WSClient, val capiClientForFrontsSeo: Co implicit val fapiClient: ApiClient = FrontsApi.crossAccountClient - override def putPressedJson(path: String, json: String): Unit = S3FrontsApi.putDraftFapiPressedJson(path, json) + override def putPressedJson(path: String, json: String, pressedType: PressedPageType): Unit = S3FrontsApi.putDraftFapiPressedJson(path, json, pressedType) override def isLiveContent: Boolean = false override def collectionContentWithSnaps( @@ -80,7 +83,7 @@ trait FapiFrontPress extends Logging { implicit def fapiClient: ApiClient val capiClientForFrontsSeo: ContentApiClient val wsClient: WSClient - def putPressedJson(path: String, json: String): Unit + def putPressedJson(path: String, json: String, pressedType: PressedPageType): Unit def isLiveContent: Boolean def collectionContentWithSnaps( @@ -112,10 +115,9 @@ trait FapiFrontPress extends Logging { val stopWatch: StopWatch = new StopWatch val pressFuture = getPressedFrontForPath(path) - .map { pressedFront: PressedPage => - val json: String = Json.stringify(Json.toJson(pressedFront)) - FaciaPressMetrics.FrontPressContentSize.recordSample(json.getBytes.length, new DateTime()) - putPressedJson(path, json) + .map { pressedFronts: PressedPageVersions => + putPressedPage(path, pressedFronts.full, FullType) + putPressedPage(path, pressedFronts.lite, LiteType) }.fold( e => { StatusNotification.notifyFailedJob(path, isLive = isLiveContent, e) @@ -148,7 +150,19 @@ trait FapiFrontPress extends Logging { pressFuture } - def generateCollectionJsonFromFapiClient(collectionId: String)(implicit executionContext: ExecutionContext): Response[PressedCollection] = + private def putPressedPage(path: String, pressedFront: PressedPage, pressedType: PressedPageType): Unit = { + val json: String = Json.stringify(Json.toJson(pressedFront)) + + val metric: SamplerMetric = pressedType match { + case FullType => FaciaPressMetrics.FrontPressContentSize + case LiteType => FaciaPressMetrics.FrontPressContentSizeLite + } + + metric.recordSample(json.getBytes.length, new DateTime()) + putPressedJson(path, json, pressedType) + } + + def generateCollectionJsonFromFapiClient(collectionId: String)(implicit executionContext: ExecutionContext): Response[PressedCollectionVersions] = { for { collection <- FAPI.getCollection(collectionId) curated <- getCurated(collection) @@ -160,17 +174,28 @@ trait FapiFrontPress extends Logging { .contains(collection.collectionConfig.collectionType) .toOption(curated.length + backfill.length) .getOrElse(Configuration.facia.collectionCap) - val trimmedCurated = curated.take(maxStories) - val trimmedBackfill = backfill.take(maxStories - trimmedCurated.length) - PressedCollection.fromCollectionWithCuratedAndBackfill( - collection, - trimmedCurated, - trimmedBackfill, - treats + + val storyCountLite = Container.storiesCount(CollectionConfig.make(collection.collectionConfig), curated ++ backfill).getOrElse(maxStories) + + PressedCollectionVersions( + pressCollection(collection, curated, backfill, treats, storyCountLite), + pressCollection(collection, curated, backfill, treats, maxStories) ) } + } + private def pressCollection(collection: Collection, curated: List[PressedContent], backfill: List[PressedContent], treats: List[PressedContent], storyCount: Int) = { + val trimmedCurated = curated.take(storyCount) + val trimmedBackfill = backfill.take(storyCount - trimmedCurated.length) + PressedCollection.fromCollectionWithCuratedAndBackfill( + collection, + trimmedCurated, + trimmedBackfill, + treats + ) + } + private def getCurated(collection: Collection)(implicit executionContext: ExecutionContext): Response[List[PressedContent]] = { // Map initial PressedContent to enhanced content which contains pre-fetched embed content. val initialContent = collectionContentWithSnaps(collection, searchApiQuery, itemApiQuery) @@ -263,14 +288,14 @@ trait FapiFrontPress extends Logging { } } - def getPressedFrontForPath(path: String)(implicit executionContext: ExecutionContext): Response[PressedPage] = { - val collectionIds = getCollectionIdsForPath(path) - collectionIds - .flatMap(c => Response.traverse(c.map(generateCollectionJsonFromFapiClient))) - .flatMap(result => - Response.Async.Right(getFrontSeoAndProperties(path).map{ - case (seoData, frontProperties) => PressedPage(path, seoData, frontProperties, result) - })) + def getPressedFrontForPath(path: String)(implicit executionContext: ExecutionContext): Response[PressedPageVersions] = { + for { + collectionIds <- getCollectionIdsForPath(path) + pressedCollections <- Response.traverse(collectionIds.map(generateCollectionJsonFromFapiClient)) + seoWithProperties <- Response.Async.Right(getFrontSeoAndProperties(path)) + } yield seoWithProperties match { + case (seoData, frontProperties) => PressedPageVersions.fromPressedCollections(path, seoData, frontProperties, pressedCollections) + } } private def getFrontSeoAndProperties(path: String)(implicit executionContext: ExecutionContext): Future[(SeoData, FrontProperties)] = diff --git a/facia-press/conf/routes b/facia-press/conf/routes index 55ca209823d5..a80e3f10d33e 100644 --- a/facia-press/conf/routes +++ b/facia-press/conf/routes @@ -2,7 +2,8 @@ GET /_healthcheck controllers.HealthCheck.healthCheck() GET / controllers.Application.index GET /debug/config controllers.Application.showCurrentConfig -GET /pressed/live/*path controllers.Application.generateLivePressedFor(path) +GET /pressed/live/*path.lite controllers.Application.generateLiteLivePressedFor(path) +GET /pressed/live/*path controllers.Application.generateLivePressedFor(path) POST /press/live/*path controllers.Application.pressLiveForPath(path) POST /press/draft/all controllers.Application.pressDraftForAll()