SEO: keyword and contributors index pages

admin/app/Global.scala

@@ -2,9 +2,11 @@ import commercial.TravelOffersCacheJob
 import common.{AkkaAsync, Jobs, CloudWatchApplicationMetrics}
 import conf.{Configuration, Gzipper, Management}
 import dfp.DfpDataCacheJob
-import jobs.RefreshFrontsJob
+import jobs.{RebuildIndexJob, RefreshFrontsJob}
 import model.AdminLifecycle
 import ophan.SurgingContentAgentLifecycle
+import play.api.Play
+import play.api.Play.current
 import play.api.mvc.{WithFilters, Results, RequestHeader}
 import scala.concurrent.Future
 
@@ -17,6 +19,8 @@ with SurgingContentAgentLifecycle {
 
   val adminPressJobPushRateInMinutes: Int = Configuration.faciatool.adminPressJobPushRateInMinutes
 
+  val adminRebuildIndexRateInMinutes: Int = Configuration.indexes.adminRebuildIndexRateInMinutes
+
   override def onError(request: RequestHeader, ex: Throwable) = Future.successful(InternalServerError(
     views.html.errorPage(ex)
   ))
@@ -27,6 +31,10 @@ with SurgingContentAgentLifecycle {
       RefreshFrontsJob.run()
     }
 
+    Jobs.schedule("RebuildIndexJob", s"0 0/$adminRebuildIndexRateInMinutes * 1/1 * ? *") {
+      RebuildIndexJob.run()
+    }
+
     // every 30 minutes
     Jobs.schedule("DfpDataCacheJob", "0 1/30 * * * ? *") {
       DfpDataCacheJob.run()
@@ -41,17 +49,24 @@ with SurgingContentAgentLifecycle {
 
   override def onStart(app: play.api.Application) {
     super.onStart(app)
-    descheduleJobs()
-    scheduleJobs()
 
-    AkkaAsync {
-      DfpDataCacheJob.run()
-      TravelOffersCacheJob.run()
+    if (!Play.isTest) {
+      descheduleJobs()
+      scheduleJobs()
+
+      AkkaAsync {
+        RebuildIndexJob.run()
+        DfpDataCacheJob.run()
+        TravelOffersCacheJob.run()
+      }
     }
   }
 
   override def onStop(app: play.api.Application) {
-    descheduleJobs()
+    if (!Play.isTest) {
+      descheduleJobs()
+    }
+
     super.onStop(app)
   }
 }

admin/app/indexes/ContentApiTagsEnumerator.scala

@@ -0,0 +1,55 @@
+package indexes
+
+import common.Logging
+import conf.LiveContentApi
+
+import scala.concurrent.Future
+import com.gu.openplatform.contentapi.model.{Tag, TagsResponse}
+import scala.concurrent.duration._
+import scala.concurrent.ExecutionContext.Implicits.global
+import play.api.libs.iteratee.{Enumeratee, Enumerator}
+
+object ContentApiTagsEnumerator extends Logging {
+  val DelayBetweenRetries = 100.millis
+  val MaxNumberRetries = 5
+  val MaxPageSize = 1000
+
+  def enumeratePages(getPage: Int => Future[TagsResponse]): Enumerator[Tag] = {
+    def getPageWithRetries(page: Int, retriesRemaining: Int = MaxNumberRetries): Future[TagsResponse] =
+      if (retriesRemaining == 0)
+        getPage(page)
+      else
+        getPage(page) recoverWith {
+          case error: Throwable =>
+            log.error(s"Error getting tag page $page, $retriesRemaining retries remaining", error)
+            getPageWithRetries(page, retriesRemaining - 1)
+        }
+
+    Enumerator.unfoldM(Option(1)) {
+      case Some(nextPage) => getPageWithRetries(nextPage) map { response =>
+        val next = if (response.isLastPage) None else Some(response.currentPage + 1)
+
+        Some(next, response.results)
+      }
+
+      case None => Future.successful(None)
+    }.flatMap(Enumerator.apply(_: _*))
+  }
+
+  def enumerateTagType(tagType: String) = enumeratePages { page =>
+    LiveContentApi.tags.tagType(tagType).pageSize(MaxPageSize).page(page).response
+  }
+
+  implicit class RichTag(tag: Tag) {
+    def isSectionTag = tag.id.split("/").toList match {
+      case first :: second :: Nil => first == second
+      case _ => false
+    }
+  }
+
+  def enumerateTagTypeFiltered(tagType: String) =
+    enumerateTagType(tagType) through Enumeratee.filter({ tag =>
+      /** Believe it or not, we actually have tags whose titles start with HTML tags ... */
+      !tag.id.startsWith("weather/") && tag.webTitle.charAt(0).isLetterOrDigit && !tag.isSectionTag
+    })
+}

admin/app/indexes/TagPages.scala

@@ -0,0 +1,111 @@
+package indexes
+
+import common.Maps._
+import com.gu.openplatform.contentapi.model.Tag
+import java.text.Normalizer
+import model.{TagDefinition, TagIndexPage}
+
+import play.api.libs.iteratee.{Enumeratee, Iteratee}
+import scala.concurrent.ExecutionContext.Implicits.global
+
+object TagPages {
+  /** To be curated by Peter Martin */
+  val ValidSections = Map(
+    ("artanddesign", "Art and design"),
+    ("better-business", "Better Business"),
+    ("books", "Books"),
+    ("business", "Business"),
+    ("cardiff", "Cardiff"),
+    ("cities", "Cities"),
+    ("commentisfree", "Comment is free"),
+    ("community", "Community"),
+    ("crosswords", "Crosswords"),
+    ("culture", "Culture"),
+    ("culture-network", "Culture Network"),
+    ("culture-professionals-network", "Culture professionals network"),
+    ("edinburgh", "Edinburgh"),
+    ("education", "Education"),
+    ("enterprise-network", "Guardian Enterprise Network"),
+    ("environment", "Environment"),
+    ("extra", "Extra"),
+    ("fashion", "Fashion"),
+    ("film", "Film"),
+    ("football", "Football"),
+    ("global-development", "Global development"),
+    ("global-development-professionals-network", "Global Development Professionals Network"),
+    ("government-computing-network", "Guardian Government Computing"),
+    ("guardian-professional", "Guardian Professional"),
+    ("healthcare-network", "Healthcare Professionals Network"),
+    ("help", "Help"),
+    ("higher-education-network", "Higher Education Network"),
+    ("housing-network", "Housing Network"),
+    ("info", "Info"),
+    ("katine", "Katine"),
+    ("law", "Law"),
+    ("leeds", "Leeds"),
+    ("lifeandstyle", "Life and style"),
+    ("local", "Local"),
+    ("local-government-network", "Local Leaders Network"),
+    ("media", "Media"),
+    ("media-network", "Media Network"),
+    ("money", "Money"),
+    ("music", "Music"),
+    ("news", "News"),
+    ("politics", "Politics"),
+    ("public-leaders-network", "Public Leaders Network"),
+    ("science", "Science"),
+    ("search", "Search"),
+    ("small-business-network", "Guardian Small Business Network"),
+    ("social-care-network", "Social Care Network"),
+    ("social-enterprise-network", "Social Enterprise Network"),
+    ("society", "Society"),
+    ("society-professionals", "Society Professionals"),
+    ("sport", "Sport"),
+    ("stage", "Stage"),
+    ("teacher-network", "Teacher Network"),
+    ("technology", "Technology"),
+    ("theguardian", "From the Guardian"),
+    ("theobserver", "From the Observer"),
+    ("travel", "Travel"),
+    ("travel/offers", "Guardian holiday offers"),
+    ("tv-and-radio", "Television & radio"),
+    ("uk-news", "UK news"),
+    ("voluntary-sector-network", "Voluntary Sector Network"),
+    ("weather", "Weather"),
+    ("women-in-leadership", "Women in Leadership"),
+    ("world", "World news")
+  )
+
+  def asAscii(s: String) =
+    Normalizer.normalize(s, Normalizer.Form.NFD).replaceAll("[^\\p{ASCII}]", "")
+
+  def alphaIndexKey(s: String) = {
+    val firstChar = asAscii(s).toLowerCase.charAt(0)
+
+    if (firstChar.isDigit) {
+      "1-9"
+    } else {
+      firstChar.toString
+    }
+  }
+
+  private def mappedByKey(key: Tag => String) =
+    Iteratee.fold[Tag, Map[String, Set[Tag]]](Map.empty) { (acc, tag) =>
+      insertWith(acc, key(tag), Set(tag))(_ union _)
+    }
+
+  def toPages(tagsByKey: Map[String, Set[Tag]])(titleFromKey: String => String) = tagsByKey.toSeq.sortBy(_._1) map { case (id, tagSet) =>
+    TagIndexPage(
+      id,
+      titleFromKey(id),
+      tagSet.toSeq.sortBy(tag => asAscii(tag.webTitle)).map(TagDefinition.fromContentApiTag)
+    )
+  }
+
+  val invalidSectionsFilter = Enumeratee.filter[Tag](_.sectionId.exists(ValidSections.contains))
+
+  val byWebTitle = mappedByKey(tag => alphaIndexKey(tag.webTitle))
+
+  val bySection = invalidSectionsFilter &>> mappedByKey(_.sectionId.get)
+}
+

admin/app/indexes/package.scala

@@ -0,0 +1,7 @@
+import com.gu.openplatform.contentapi.model.TagsResponse
+
+package object indexes {
+  implicit class RichTagsResponse(tagsResponse: TagsResponse) {
+    def isLastPage = tagsResponse.currentPage >= tagsResponse.pages
+  }
+}

admin/app/jobs/RebuildIndexJob.scala

@@ -0,0 +1,71 @@
+package jobs
+
+import common.{Logging, ExecutionContexts, StopWatch}
+import indexes.ContentApiTagsEnumerator
+import indexes.TagPages._
+import model.{TagIndexListings, TagIndexPage}
+import play.api.libs.iteratee.Enumeratee
+import services.TagIndexesS3
+
+import scala.concurrent.{Future, blocking}
+
+object RebuildIndexJob extends ExecutionContexts with Logging {
+  def saveToS3(parentKey: String, tagPages: Seq[TagIndexPage]) {
+    val s3StopWatch = new StopWatch
+
+    tagPages foreach { tagPage =>
+      log.info(s"Uploading $parentKey ${tagPage.title} index to S3")
+      TagIndexesS3.putIndex(parentKey, tagPage)
+    }
+
+    log.info(s"Uploaded ${tagPages.length} $parentKey index pages to S3 after ${s3StopWatch.elapsed}ms")
+
+    val listingStopWatch = new StopWatch
+
+    TagIndexesS3.putListing(parentKey, TagIndexListings.fromTagIndexPages(tagPages))
+
+    log.info(s"Uploaded $parentKey listing in ${listingStopWatch.elapsed}ms")
+  }
+
+  /** The title for the alpha keys (A, B, C ... )
+    *
+    * Replace the hyphen with an ndash here as it looks better in the HTML. (The key needs to be a hyphen though so it
+    * works in a web uri.)
+    */
+  private def alphaTitle(key: String) = key.toUpperCase.replace("-", "–")
+
+  def rebuildKeywordIndexes() = {
+    /** Keywords are indexed both alphabetically and by their parent section */
+    ContentApiTagsEnumerator.enumerateTagTypeFiltered("keyword")
+      .run(Enumeratee.zip(bySection, byWebTitle)) map { case (sectionMap, alphaMap) =>
+        blocking {
+          saveToS3("keywords", toPages(alphaMap)(alphaTitle))
+          saveToS3("keywords_by_section", toPages(sectionMap)(ValidSections(_)))
+        }
+    }
+  }
+
+  def rebuildContributorIndex() = {
+    ContentApiTagsEnumerator.enumerateTagTypeFiltered("contributor")
+      .run(byWebTitle) map { alphaMap =>
+      blocking {
+        saveToS3("contributors", toPages(alphaMap)(alphaTitle))
+      }
+    }
+  }
+
+  implicit class RichFuture[A](future: Future[A]) {
+    def withErrorLogging = {
+      future onFailure {
+        case throwable: Throwable =>
+          log.error("Error rebuilding index", throwable)
+      }
+
+      future
+    }
+  }
+
+  def run() {
+    rebuildKeywordIndexes().withErrorLogging andThen { case _ => rebuildContributorIndex().withErrorLogging }
+  }
+}