From 3ffecd12d1f8e800fb99800f6beccdd89df95fff Mon Sep 17 00:00:00 2001
From: Murilo Dal Ri <murilodalri@gmail.com>
Date: Thu, 17 Oct 2024 14:35:33 +0100
Subject: [PATCH] Move logs from govuk-aws

Move the infra-fastly-logs project from govuk-aws.
---
 logs/.terraform.lock.hcl       |  62 +++
 logs/integration.govuk.backend |   4 +
 logs/main.tf                   | 955 +++++++++++++++++++++++++++++++++
 logs/main_imports.tf           | 154 ++++++
 logs/production.govuk.backend  |   4 +
 logs/staging.govuk.backend     |   4 +
 logs/transition_logs.py        |  40 ++
 logs/variables.tf              |  10 +
 8 files changed, 1233 insertions(+)
 create mode 100644 logs/.terraform.lock.hcl
 create mode 100644 logs/integration.govuk.backend
 create mode 100644 logs/main.tf
 create mode 100644 logs/main_imports.tf
 create mode 100644 logs/production.govuk.backend
 create mode 100644 logs/staging.govuk.backend
 create mode 100644 logs/transition_logs.py
 create mode 100644 logs/variables.tf

diff --git a/logs/.terraform.lock.hcl b/logs/.terraform.lock.hcl
new file mode 100644
index 0000000..a6a1e2c
--- /dev/null
+++ b/logs/.terraform.lock.hcl
@@ -0,0 +1,62 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/archive" {
+  version = "2.6.0"
+  hashes = [
+    "h1:Ou6XKWvpo7IYgZnrFJs5MKzMqQMEYv8Z2iHSJ2mmnFw=",
+    "zh:29273484f7423b7c5b3f5df34ccfc53e52bb5e3d7f46a81b65908e7a8fd69072",
+    "zh:3cba58ec3aea5f301caf2acc31e184c55d994cc648126cac39c63ae509a14179",
+    "zh:55170cd17dbfdea842852c6ae2416d057fec631ba49f3bb6466a7268cd39130e",
+    "zh:7197db402ba35631930c3a4814520f0ebe980ae3acb7f8b5a6f70ec90dc4a388",
+    "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+    "zh:8bf7fe0915d7fb152a3a6b9162614d2ec82749a06dba13fab3f98d33c020ec4f",
+    "zh:8ce811844fd53adb0dabc9a541f8cb43aacfa7d8e39324e4bd3592b3428f5bfb",
+    "zh:bca795bca815b8ac90e3054c0a9ab1ccfb16eedbb3418f8ad473fc5ad6bf0ef7",
+    "zh:d9355a18df5a36cf19580748b23249de2eb445c231c36a353709f8f40a6c8432",
+    "zh:dc32cc32cfd8abf8752d34f2a783de0d3f7200c573b885ecb64ece5acea173b4",
+    "zh:ef498e20391bf7a280d0fd6fd6675621c85fbe4e92f0f517ae4394747db89bde",
+    "zh:f2bc5226c765b0c8055a7b6207d0fe1eb9484e3ec8880649d158827ac6ed3b22",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version = "5.67.0"
+  hashes = [
+    "h1:gljTHIfOelTepL5K1zblNXb3yaUDxcZTEyXeMvO+H1E=",
+    "zh:1259c8106c0a3fc0ed3b3eb814ab88d6a672e678b533f47d1bbbe3107949f43e",
+    "zh:226414049afd6d334cc16ff5d6cef23683620a9b56da67a21422a113d9cce4ab",
+    "zh:3c89b103aea20ef82a84e889abaeb971cb168de8292b61b34b83e807c40085a9",
+    "zh:3dd88e994fb7d7a6c6eafd3c01393274e4f776021176acea2e980f73fbd4acbc",
+    "zh:487e0dda221c84a20a143904c1cee4e63fce6c5c57c21368ea79beee87b108da",
+    "zh:7693bdcec8181aafcbda2c41c35b1386997e2c92b6f011df058009e4c8b300e1",
+    "zh:82679536250420f9e8e6edfd0fa9a1bab99a7f31fe5f049ac7a2e0d8c287b56f",
+    "zh:8685218dae921740083820c52afa66cdf14cf130539da1efd7d9a78bfb6ade64",
+    "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+    "zh:9e553a3ec05eedea779d393447fc316689ba6c4d4d8d569b986898e6dbe58fee",
+    "zh:a36c24acd3c75bac8211fefde58c459778021eb871ff8339be1c26ad8fd67ee1",
+    "zh:ce48bd1e35d6f996f1a09d8f99e8084469b7fec5611e67a50a63e96375b87ebe",
+    "zh:d6c76a24205513725269e4783da14be9648e9086fb621496052f4b37d52d785e",
+    "zh:d95a31745affb178ea48fa8e0be94691a8f7507ea55c0d0a4b6e0a8ef6fcb929",
+    "zh:f061ce59fac1bc425c1092e6647ed4bb1b61824416041b46dbf336e01a63ad89",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/tfe" {
+  version = "0.59.0"
+  hashes = [
+    "h1:c6W/8LNZ4zwyBmNFiZ1iSQny6kDV9HnXdomACAhvexw=",
+    "zh:39d4513f023d57c83f3d1e659db6093dd661ef34f7b5d8025b8155a19244e8e4",
+    "zh:4c2183c32356c0345c86b063c0bc9276fe704c9fc5c94ba9b45394de9d22a08a",
+    "zh:55ec15a226636e48dc17bad05781c2f679c9fd40940aff3f8109e6c2d4869e3e",
+    "zh:6cac67993b64000fd7a9856c82115e3af9ebf2ebe5e6c9870865cbb1da5a8b8e",
+    "zh:6ea98ea1e4afd42b44a84756914ee80dbc6f81f0aa314def7316c9c14b8114b0",
+    "zh:a708a6bdc80cf77793c2e9d367f628c3fdd8b1ec05f7b08ffb8ef4944d625a23",
+    "zh:a960e9330494d63a796ed71e3563b13121115f02806dfea8c64231753dac53bd",
+    "zh:b752c5c1000728b22cda47b5032ae537a46a8e5c197642102cb3e5dab95ce090",
+    "zh:c16b0899b984ef398b2b234d4171cab17484c8eeb7f20cc0eaea172d337ae187",
+    "zh:d95ec293fa70e946b6cd657912b33155f8be3413e6128ed2bfa5a493f788e439",
+    "zh:ddb8ddcf0cc945f369e537285c727be9c673477ca62d7f9800287041539841aa",
+    "zh:e917a03a99b8f3ca1021a099e421824e8e98fc63913de2932bb0a7dd54b24461",
+  ]
+}
diff --git a/logs/integration.govuk.backend b/logs/integration.govuk.backend
new file mode 100644
index 0000000..7c5ec12
--- /dev/null
+++ b/logs/integration.govuk.backend
@@ -0,0 +1,4 @@
+bucket  = "govuk-terraform-steppingstone-integration"
+key     = "govuk/infra-fastly-logs.tfstate"
+encrypt = true
+region  = "eu-west-1"
diff --git a/logs/main.tf b/logs/main.tf
new file mode 100644
index 0000000..62c4d0c
--- /dev/null
+++ b/logs/main.tf
@@ -0,0 +1,955 @@
+terraform {
+  cloud {
+    organization = "govuk"
+    workspaces {
+      tags = ["logs", "fastly"]
+    }
+  }
+  required_version = "~> 1.5"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 5.0"
+    }
+  }
+}
+
+provider "aws" {
+  region = var.aws_region
+}
+
+provider "tfe" {}
+
+provider "archive" {}
+
+resource "aws_s3_bucket" "fastly_logs" {
+  bucket = "govuk-${var.environment}-fastly-logs"
+}
+
+resource "aws_s3_bucket_logging" "fastly_logs" {
+  bucket = aws_s3_bucket.fastly_logs.id
+
+  target_bucket = "govuk-${var.environment}-aws-logging"
+  target_prefix = "s3/govuk-${var.environment}-fastly-logs/"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "fastly_logs" {
+  bucket = aws_s3_bucket.fastly_logs.id
+
+  rule {
+    id = "expire"
+
+    status = "Enabled"
+
+    expiration {
+      days = 120
+    }
+
+    noncurrent_version_expiration {
+      noncurrent_days           = 1
+      newer_noncurrent_versions = ""
+    }
+  }
+}
+
+# We require a user for Fastly to write to S3 buckets
+resource "aws_iam_user" "logs_writer" {
+  name = "govuk-${var.environment}-fastly-logs-writer"
+}
+
+resource "aws_iam_policy" "logs_writer" {
+  name        = "fastly-logs-${var.environment}-logs-writer-policy"
+  policy      = <<EOF
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket",
+                "s3:ListAllMyBuckets",
+                "s3:GetBucketLocation"
+             ],
+            "Resource": [
+                "arn:aws:s3:::*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:GetObject",
+                "s3:ListBucket",
+                "s3:PutObject"
+            ],
+            "Resource": [
+                "arn:aws:s3:::${aws_s3_bucket.fastly_logs.id}",
+                "arn:aws:s3:::${aws_s3_bucket.fastly_logs.id}/*"
+            ]
+        }
+    ]
+}
+EOF
+  description = "Allows writing to to the fastly-logs bucket"
+}
+
+resource "aws_iam_policy_attachment" "logs_writer" {
+  name       = "logs-writer-policy-attachment"
+  users      = ["${aws_iam_user.logs_writer.name}"]
+  policy_arn = aws_iam_policy.logs_writer.arn
+}
+
+resource "aws_glue_catalog_database" "fastly_logs" {
+  name        = "fastly_logs"
+  description = "Used to browse the CDN log files that Fastly sends"
+}
+
+resource "aws_iam_role_policy_attachment" "aws-glue-service-role-service-attachment" {
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
+  role       = aws_iam_role.glue.name
+}
+
+resource "aws_iam_role" "glue" {
+  name = "AWSGlueServiceRole-fastly-logs"
+
+  // I did want to set a path of /service-role/ here but that seems to break
+  // creating the crawler
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "glue.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "fastly_logs_policy" {
+  name = "govuk-${var.environment}-fastly-logs-glue-policy"
+  role = aws_iam_role.glue.id
+
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:*"
+      ],
+      "Resource": [
+        "arn:aws:s3:::${aws_s3_bucket.fastly_logs.id}",
+        "arn:aws:s3:::${aws_s3_bucket.fastly_logs.id}/*"
+      ]
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_glue_crawler" "govuk_www" {
+  name          = "GOV.UK fastly logs"
+  description   = "Crawls the GOV.UK logs from fastly for allowing Athena querying"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  role          = aws_iam_role.glue.name
+  schedule      = "cron(30 */4 * * ? *)"
+
+  s3_target {
+    path = "s3://${aws_s3_bucket.fastly_logs.bucket}/govuk_www"
+  }
+
+  schema_change_policy {
+    delete_behavior = "DELETE_FROM_DATABASE"
+    update_behavior = "LOG"
+  }
+
+  configuration = <<EOF
+{
+  "Version": 1.0,
+  "CrawlerOutput": {
+    "Partitions": {
+      "AddOrUpdateBehavior": "InheritFromTable"
+    }
+  }
+}
+EOF
+}
+
+resource "aws_glue_catalog_table" "govuk_www" {
+  name          = "govuk_www"
+  description   = "Allows access to JSON data exported from Fastly"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  table_type    = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    compressed    = true
+    location      = "s3://${aws_s3_bucket.fastly_logs.bucket}/govuk_www/"
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name = "ser_de_name"
+
+      parameters = {
+        paths                   = "client_ip,request_received,request_received_offset,method,url,status,protocol,request_time,time_to_generate_response,bytes,content_type,user_agent,fastly_backend,data_centre,cache_hit,cache_response,tls_client_protocol,tls_client_cipher,client_ja3"
+        "ignore.malformed.json" = "true"
+      }
+
+      serialization_library = "org.openx.data.jsonserde.JsonSerDe"
+    }
+
+    // These columns correlate with the log format set up in Fastly as below
+    //
+    // {
+    // "client_ip":"%{json.escape(client.ip)}V",
+    // "request_received":"%{begin:%Y-%m-%d %H:%M:%S.}t%{time.start.msec_frac}V",
+    // "request_received_offset":"%{begin:%z}t",
+    // "method":"%{json.escape(req.method)}V",
+    // "url":"%{json.escape(req.url)}V",
+    // "status":%>s,
+    // "protocol":"%{json.escape(req.proto)}V",
+    // "request_time":%{time.elapsed.sec}V.%{time.elapsed.msec_frac}V,
+    // "time_to_generate_response":%{time.to_first_byte}V,
+    // "bytes":%B,
+    // "content_type":"%{json.escape(resp.http.Content-Type)}V",
+    // "user_agent":"%{json.escape(req.http.User-Agent)}V",
+    // "fastly_backend":"%{json.escape(resp.http.Fastly-Backend-Name)}V",
+    // "data_centre":"%{json.escape(server.datacenter)}V",
+    // "cache_hit":%{if(fastly_info.state ~"^(HIT|MISS)(?:-|$)", "true", "false")}V,
+    // "cache_response":"%{regsub(fastly_info.state, "^(HIT-(SYNTH)|(HITPASS|HIT|MISS|PASS|ERROR|PIPE)).*", "\\2\\3") }V",
+    // "tls_client_protocol":"%{json.escape(tls.client.protocol)}V",
+    // "tls_client_cipher":"%{json.escape(tls.client.cipher)}V",
+    // "client_ja3":"%{json.escape(req.http.Client-JA3)}V"
+    // }
+    columns {
+      name    = "client_ip"
+      type    = "string"
+      comment = "IP address of the client that made the request"
+    }
+    columns {
+      name    = "request_received"
+      type    = "timestamp"
+      comment = "Time we received the request"
+    }
+    columns {
+      // This field is separate from the timestamp above as the Presto version
+      // on AWS Athena doesn't support timestamps - expectation is that this is
+      // always +0000 though
+      name = "request_received_offset"
+
+      type    = "string"
+      comment = "Time offset of the request, expected to be +0000 always"
+    }
+    columns {
+      name    = "method"
+      type    = "string"
+      comment = "HTTP method for this request"
+    }
+    columns {
+      name    = "url"
+      type    = "string"
+      comment = "URL requested with query string"
+    }
+    columns {
+      name    = "status"
+      type    = "int"
+      comment = "HTTP status code returned"
+    }
+    columns {
+      name    = "request_time"
+      type    = "double"
+      comment = "Time until user received full response in seconds"
+    }
+    columns {
+      name    = "time_to_generate_response"
+      type    = "double"
+      comment = "Time spent generating a response for varnish, in seconds"
+    }
+    columns {
+      name    = "bytes"
+      type    = "bigint"
+      comment = "Number of bytes returned"
+    }
+    columns {
+      name    = "content_type"
+      type    = "string"
+      comment = "HTTP Content-Type header returned"
+    }
+    columns {
+      name    = "user_agent"
+      type    = "string"
+      comment = "User agent that made the request"
+    }
+    columns {
+      name    = "fastly_backend"
+      type    = "string"
+      comment = "Name of the backend that served this request"
+    }
+    columns {
+      name    = "data_centre"
+      type    = "string"
+      comment = "Name of the data centre that served this request"
+    }
+    columns {
+      name    = "cache_hit"
+      type    = "boolean"
+      comment = "Whether this object is cacheable or not"
+    }
+    columns {
+      name    = "cache_response"
+      type    = "string"
+      comment = "Whether the response was a HIT, MISS, PASS, ERROR, PIPE, HITPASS, or SYNTH(etic)"
+    }
+    columns {
+      name = "tls_client_protocol"
+      type = "string"
+    }
+    columns {
+      name = "tls_client_cipher"
+      type = "string"
+    }
+    columns {
+      name = "client_ja3"
+      type = "string"
+    }
+    columns {
+      name    = "protocol"
+      type    = "string"
+      comment = "HTTP version used"
+    }
+  }
+
+  // these correspond to directory ordering of:
+  // /year=YYYY/month=MM/date=DD/file.log.gz
+  partition_keys {
+    name = "year"
+    type = "int"
+  }
+  partition_keys {
+    name = "month"
+    type = "int"
+  }
+  partition_keys {
+    name = "date"
+    type = "int"
+  }
+
+  parameters = {
+    classification  = "json"
+    compressionType = "gzip"
+    typeOfDate      = "file"
+  }
+}
+
+resource "aws_glue_crawler" "govuk_assets" {
+  name          = "Assets fastly logs"
+  description   = "Crawls the assets logs from fastly for allowing Athena querying"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  role          = aws_iam_role.glue.name
+  schedule      = "cron(30 */4 * * ? *)"
+
+  s3_target {
+    path = "s3://${aws_s3_bucket.fastly_logs.bucket}/govuk_assets"
+  }
+
+  schema_change_policy {
+    delete_behavior = "DELETE_FROM_DATABASE"
+    update_behavior = "LOG"
+  }
+
+  configuration = <<EOF
+{
+  "Version": 1.0,
+  "CrawlerOutput": {
+    "Partitions": {
+      "AddOrUpdateBehavior": "InheritFromTable"
+    }
+  }
+}
+EOF
+}
+
+resource "aws_glue_catalog_table" "govuk_assets" {
+  name          = "govuk_assets"
+  description   = "Allows access to JSON data exported from Fastly"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  table_type    = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    compressed    = true
+    location      = "s3://${aws_s3_bucket.fastly_logs.bucket}/govuk_assets/"
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name = "ser_de_name"
+
+      parameters = {
+        paths                   = "client_ip,request_received,request_received_offset,method,url,status,protocol,request_time,time_to_generate_response,bytes,content_type,user_agent,fastly_backend,data_centre,cache_hit,cache_response,tls_client_protocol,tls_client_cipher,client_ja3"
+        "ignore.malformed.json" = "true"
+      }
+
+      serialization_library = "org.openx.data.jsonserde.JsonSerDe"
+    }
+
+    // These columns correlate with the log format set up in Fastly as below
+    //
+    // {
+    // "client_ip":"%{json.escape(client.ip)}V",
+    // "request_received":"%{begin:%Y-%m-%d %H:%M:%S.}t%{time.start.msec_frac}V",
+    // "request_received_offset":"%{begin:%z}t",
+    // "method":"%{json.escape(req.method)}V",
+    // "url":"%{json.escape(req.url)}V",
+    // "status":%>s,
+    // "protocol":"%{json.escape(req.proto)}V",
+    // "request_time":%{time.elapsed.sec}V.%{time.elapsed.msec_frac}V,
+    // "time_to_generate_response":%{time.to_first_byte}V,
+    // "bytes":%B,
+    // "content_type":"%{json.escape(resp.http.Content-Type)}V",
+    // "user_agent":"%{json.escape(req.http.User-Agent)}V",
+    // "fastly_backend":"%{json.escape(resp.http.Fastly-Backend-Name)}V",
+    // "data_centre":"%{json.escape(server.datacenter)}V",
+    // "cache_hit":%{if(fastly_info.state ~"^(HIT|MISS)(?:-|$)", "true", "false")}V,
+    // "cache_response":"%{regsub(fastly_info.state, "^(HIT-(SYNTH)|(HITPASS|HIT|MISS|PASS|ERROR|PIPE)).*", "\\2\\3") }V",
+    // "tls_client_protocol":"%{json.escape(tls.client.protocol)}V",
+    // "tls_client_cipher":"%{json.escape(tls.client.cipher)}V",
+    // "client_ja3":"%{json.escape(req.http.Client-JA3)}V"
+    // }
+    columns {
+      name    = "client_ip"
+      type    = "string"
+      comment = "IP address of the client that made the request"
+    }
+    columns {
+      name    = "request_received"
+      type    = "timestamp"
+      comment = "Time we received the request"
+    }
+    columns {
+      // This field is separate from the timestamp above as the Presto version
+      // on AWS Athena doesn't support timestamps - expectation is that this is
+      // always +0000 though
+      name = "request_received_offset"
+
+      type    = "string"
+      comment = "Time offset of the request, expected to be +0000 always"
+    }
+    columns {
+      name    = "method"
+      type    = "string"
+      comment = "HTTP method for this request"
+    }
+    columns {
+      name    = "url"
+      type    = "string"
+      comment = "URL requested with query string"
+    }
+    columns {
+      name    = "status"
+      type    = "int"
+      comment = "HTTP status code returned"
+    }
+    columns {
+      name    = "request_time"
+      type    = "double"
+      comment = "Time until user received full response in seconds"
+    }
+    columns {
+      name    = "time_to_generate_response"
+      type    = "double"
+      comment = "Time spent generating a response for varnish, in seconds"
+    }
+    columns {
+      name    = "bytes"
+      type    = "bigint"
+      comment = "Number of bytes returned"
+    }
+    columns {
+      name    = "content_type"
+      type    = "string"
+      comment = "HTTP Content-Type header returned"
+    }
+    columns {
+      name    = "user_agent"
+      type    = "string"
+      comment = "User agent that made the request"
+    }
+    columns {
+      name    = "fastly_backend"
+      type    = "string"
+      comment = "Name of the backend that served this request"
+    }
+    columns {
+      name    = "data_centre"
+      type    = "string"
+      comment = "Name of the data centre that served this request"
+    }
+    columns {
+      name    = "cache_hit"
+      type    = "boolean"
+      comment = "Whether this object is cacheable or not"
+    }
+    columns {
+      name    = "cache_response"
+      type    = "string"
+      comment = "Whether the response was a HIT, MISS, PASS, ERROR, PIPE, HITPASS, or SYNTH(etic)"
+    }
+    columns {
+      name = "tls_client_protocol"
+      type = "string"
+    }
+    columns {
+      name = "tls_client_cipher"
+      type = "string"
+    }
+    columns {
+      name = "client_ja3"
+      type = "string"
+    }
+    columns {
+      name    = "protocol"
+      type    = "string"
+      comment = "HTTP version used"
+    }
+  }
+
+  // these correspond to directory ordering of:
+  // /year=YYYY/month=MM/date=DD/file.log.gz
+  partition_keys {
+    name = "year"
+    type = "int"
+  }
+  partition_keys {
+    name = "month"
+    type = "int"
+  }
+  partition_keys {
+    name = "date"
+    type = "int"
+  }
+
+  parameters = {
+    classification  = "json"
+    compressionType = "gzip"
+    typeOfDate      = "file"
+  }
+}
+
+resource "aws_glue_crawler" "bouncer" {
+  name          = "Bouncer fastly logs"
+  description   = "Crawls the bouncer logs from fastly for allowing Athena querying"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  role          = aws_iam_role.glue.name
+  schedule      = "cron(30 */4 * * ? *)"
+
+  s3_target {
+    path = "s3://${aws_s3_bucket.fastly_logs.bucket}/bouncer"
+  }
+
+  schema_change_policy {
+    delete_behavior = "DELETE_FROM_DATABASE"
+    update_behavior = "LOG"
+  }
+
+  configuration = <<EOF
+{
+  "Version": 1.0,
+  "CrawlerOutput": {
+    "Partitions": {
+      "AddOrUpdateBehavior": "InheritFromTable"
+    }
+  }
+}
+EOF
+}
+
+resource "aws_glue_catalog_table" "bouncer" {
+  name          = "bouncer"
+  description   = "Allows access to JSON data exported from Fastly"
+  database_name = aws_glue_catalog_database.fastly_logs.name
+  table_type    = "EXTERNAL_TABLE"
+
+  storage_descriptor {
+    compressed    = true
+    location      = "s3://${aws_s3_bucket.fastly_logs.bucket}/bouncer/"
+    input_format  = "org.apache.hadoop.mapred.TextInputFormat"
+    output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
+
+    ser_de_info {
+      name = "ser_de_name"
+
+      parameters = {
+        paths                   = "client_ip,request_received,request_received_offset,method,url,status,request_time,time_to_generate_response,content_type,user_agent,data_centre,cache_hit,cache_response"
+        "ignore.malformed.json" = "true"
+      }
+
+      serialization_library = "org.openx.data.jsonserde.JsonSerDe"
+    }
+
+    // These columns correlate with the log format set up in Fastly as below
+    //
+    // {
+    // "client_ip":"%{json.escape(client.ip)}V",
+    // "request_received":"%{begin:%Y-%m-%d %H:%M:%S.}t%{time.start.msec_frac}V",
+    // "request_received_offset":"%{begin:%z}t",
+    // "method":"%{json.escape(req.method)}V",
+    // "host":"%{json.escape(req.http.host)}V",
+    // "url":"%{json.escape(req.url)}V",
+    // "status":%>s,
+    // "request_time":%{time.elapsed.sec}V.%{time.elapsed.msec_frac}V,
+    // "time_to_generate_response":%{time.to_first_byte}V,
+    // "location":"%{json.escape(resp.http.Location)}V",
+    // "user_agent":"%{json.escape(req.http.User-Agent)}V",
+    // "data_centre":"%{json.escape(server.datacenter)}V",
+    // "cache_hit":%{if(fastly_info.state ~"^(HIT|MISS)(?:-|$)", "true", "false")}V,
+    // "cache_response":"%{regsub(fastly_info.state, "^(HIT-(SYNTH)|(HITPASS|HIT|MISS|PASS|ERROR|PIPE)).*", "\\2\\3") }V"
+    // }
+    columns {
+      name    = "client_ip"
+      type    = "string"
+      comment = "IP address of the client that made the request"
+    }
+    columns {
+      name    = "request_received"
+      type    = "timestamp"
+      comment = "Time we received the request"
+    }
+    columns {
+      // This field is separate from the timestamp above as the Presto version
+      // on AWS Athena doesn't support timestamps - expectation is that this is
+      // always +0000 though
+      name = "request_received_offset"
+
+      type    = "string"
+      comment = "Time offset of the request, expected to be +0000 always"
+    }
+    columns {
+      name    = "method"
+      type    = "string"
+      comment = "HTTP method for this request"
+    }
+    columns {
+      name    = "host"
+      type    = "string"
+      comment = "Host that was requested"
+    }
+    columns {
+      name    = "url"
+      type    = "string"
+      comment = "URL requested with query string"
+    }
+    columns {
+      name    = "status"
+      type    = "int"
+      comment = "HTTP status code returned"
+    }
+    columns {
+      name    = "request_time"
+      type    = "double"
+      comment = "Time until user received full response in seconds"
+    }
+    columns {
+      name    = "time_to_generate_response"
+      type    = "double"
+      comment = "Time spent generating a response for varnish, in seconds"
+    }
+    columns {
+      name    = "location"
+      type    = "string"
+      comment = "HTTP Location header returned"
+    }
+    columns {
+      name    = "user_agent"
+      type    = "string"
+      comment = "User agent that made the request"
+    }
+    columns {
+      name    = "data_centre"
+      type    = "string"
+      comment = "Name of the data centre that served this request"
+    }
+    columns {
+      name    = "cache_hit"
+      type    = "boolean"
+      comment = "Whether this object is cacheable or not"
+    }
+    columns {
+      name    = "cache_response"
+      type    = "string"
+      comment = "Whether the response was a HIT, MISS, PASS, ERROR, PIPE, HITPASS, or SYNTH(etic)"
+    }
+  }
+
+  // these correspond to directory ordering of:
+  // /year=YYYY/month=MM/date=DD/file.log.gz
+  partition_keys {
+    name = "year"
+    type = "int"
+  }
+  partition_keys {
+    name = "month"
+    type = "int"
+  }
+  partition_keys {
+    name = "date"
+    type = "int"
+  }
+
+  parameters = {
+    classification  = "json"
+    compressionType = "gzip"
+    typeOfDate      = "file"
+  }
+}
+
+# Configuration for monitoring the fastly logs Athena databases continue to be
+# queryable. This requires a dedicated user that can query athena and save
+# the queries results
+
+resource "aws_s3_bucket" "fastly_logs_monitoring" {
+  bucket = "govuk-${var.environment}-fastly-logs-monitoring"
+}
+
+resource "aws_s3_bucket_logging" "fastly_logs_monitoring" {
+  bucket = aws_s3_bucket.fastly_logs_monitoring.id
+
+  target_bucket = "govuk-${var.environment}-aws-logging"
+  target_prefix = "s3/govuk-${var.environment}-fastly-logs-monitoring/"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "fastly_logs_monitoring" {
+  bucket = aws_s3_bucket.fastly_logs_monitoring.id
+
+  rule {
+    id = "expire"
+
+    status = "Enabled"
+
+    expiration {
+      days = 7
+    }
+  }
+}
+
+# Configuration for transition lambda function that loads data from fastly logs
+# Athena databases and saves it back into S3
+
+resource "aws_s3_bucket" "transition_fastly_logs" {
+  bucket = "govuk-${var.environment}-transition-fastly-logs"
+}
+
+resource "aws_s3_bucket_logging" "transition_fastly_logs" {
+  bucket = aws_s3_bucket.transition_fastly_logs.id
+
+  target_bucket = "govuk-${var.environment}-aws-logging"
+  target_prefix = "s3/govuk-${var.environment}-transition-fastly-logs/"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "transition_fastly_logs" {
+  bucket = aws_s3_bucket.transition_fastly_logs.id
+
+  rule {
+    id = "expire"
+
+    status = "Enabled"
+
+    expiration {
+      days = 30
+    }
+
+    noncurrent_version_expiration {
+      noncurrent_days           = 1
+      newer_noncurrent_versions = ""
+    }
+  }
+}
+
+# We require a user for transition to read from S3 buckets
+resource "aws_iam_user" "transition_downloader" {
+  name = "govuk-${var.environment}-transition-downloader"
+}
+
+resource "aws_iam_policy" "transition_downloader" {
+  name   = "fastly-logs-${var.environment}-transition-downloader-policy"
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+       "Effect":"Allow",
+       "Action":[
+          "s3:GetObject",
+          "s3:ListBucket"
+       ],
+       "Resource":[
+          "${aws_s3_bucket.transition_fastly_logs.arn}*"
+       ]
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_policy_attachment" "transition_downloader" {
+  name       = "transition-downloader-policy-attachment"
+  users      = ["${aws_iam_user.transition_downloader.name}"]
+  policy_arn = aws_iam_policy.transition_downloader.arn
+}
+
+resource "aws_athena_named_query" "transition_logs" {
+  name     = "transition-logs-query"
+  database = aws_glue_catalog_database.fastly_logs.name
+  query    = <<EOF
+SELECT
+  (current_date - interval '1' day) date,
+  count(*) count,
+  status,
+  host,
+  url
+FROM bouncer
+WHERE year = year(current_date - interval '1' day)
+  AND month = month(current_date - interval '1' day)
+  AND date = day(current_date - interval '1' day)
+GROUP BY status, host, url
+HAVING count(*) >= 10
+ORDER BY 2 DESC
+EOF
+}
+
+data "archive_file" "transition_executor" {
+  type        = "zip"
+  source_file = "${path.module}/transition_logs.py"
+  output_path = "${path.module}/TransitionLogs.zip"
+}
+
+resource "aws_lambda_function" "transition_executor" {
+  filename         = data.archive_file.transition_executor.output_path
+  source_code_hash = data.archive_file.transition_executor.output_base64sha256
+
+  function_name = "govuk-${var.environment}-transition"
+  role          = aws_iam_role.transition_executor.arn
+  handler       = "main.lambda_handler"
+  runtime       = "python3.8"
+
+  environment {
+    variables = {
+      NAMED_QUERY_ID = "${aws_athena_named_query.transition_logs.id}"
+      DATABASE_NAME  = "${aws_athena_named_query.transition_logs.database}"
+      BUCKET_NAME    = "${aws_s3_bucket.transition_fastly_logs.bucket}"
+    }
+  }
+}
+
+resource "aws_iam_role" "transition_executor" {
+  name = "AWSLambdaRole-transition-executor"
+
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": "sts:AssumeRole",
+      "Principal": {
+        "Service": "lambda.amazonaws.com"
+      },
+      "Effect": "Allow",
+      "Sid": ""
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_policy" "transition_executor" {
+  name   = "fastly-logs-${var.environment}-transition-executor-policy"
+  policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "arn:aws:logs:*:*:*",
+      "Effect": "Allow"
+    },
+    {
+       "Effect":"Allow",
+       "Action":[
+          "athena:*"
+       ],
+       "Resource":[
+          "*"
+       ]
+    },
+    {
+       "Effect":"Allow",
+       "Action":[
+          "glue:GetTable",
+          "glue:GetPartition",
+          "glue:GetPartitions"
+       ],
+       "Resource":[
+          "*"
+       ]
+    },
+    {
+       "Effect":"Allow",
+       "Action":[
+          "s3:GetObject",
+          "s3:ListBucket"
+       ],
+       "Resource":[
+          "${aws_s3_bucket.fastly_logs.arn}*"
+       ]
+    },
+    {
+       "Effect":"Allow",
+       "Action":[
+          "s3:GetBucketLocation",
+          "s3:GetObject",
+          "s3:ListBucket",
+          "s3:ListBucketMultipartUploads",
+          "s3:ListMultipartUploadParts",
+          "s3:AbortMultipartUpload",
+          "s3:PutObject"
+       ],
+       "Resource":[
+          "${aws_s3_bucket.transition_fastly_logs.arn}*"
+       ]
+    }
+  ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy_attachment" "transition_executor" {
+  role       = aws_iam_role.transition_executor.name
+  policy_arn = aws_iam_policy.transition_executor.arn
+}
+
+resource "aws_cloudwatch_event_rule" "transition_executor_daily" {
+  name                = "transition_executor_daily"
+  schedule_expression = "cron(30 0 * * ? *)"
+}
+
+resource "aws_cloudwatch_event_target" "transition_executor_daily" {
+  rule = aws_cloudwatch_event_rule.transition_executor_daily.name
+  arn  = aws_lambda_function.transition_executor.arn
+}
+
+resource "aws_lambda_permission" "cloudwatch_transition_executor_daily_permission" {
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.transition_executor.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.transition_executor_daily.arn
+}
diff --git a/logs/main_imports.tf b/logs/main_imports.tf
new file mode 100644
index 0000000..5002c1c
--- /dev/null
+++ b/logs/main_imports.tf
@@ -0,0 +1,154 @@
+import {
+  to = aws_s3_bucket.fastly_logs
+  id = "govuk-production-fastly-logs"
+}
+
+import {
+  to = aws_s3_bucket_logging.fastly_logs
+  id = "govuk-production-fastly-logs"
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.fastly_logs
+  id = "govuk-production-fastly-logs"
+}
+
+import {
+  to = aws_s3_bucket.fastly_logs_monitoring
+  id = "govuk-production-fastly-logs-monitoring"
+}
+
+import {
+  to = aws_s3_bucket_logging.fastly_logs_monitoring
+  id = "govuk-production-fastly-logs-monitoring"
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.fastly_logs_monitoring
+  id = "govuk-production-fastly-logs-monitoring"
+}
+
+import {
+  to = aws_s3_bucket.transition_fastly_logs
+  id = "govuk-production-transition-fastly-logs"
+}
+
+import {
+  to = aws_s3_bucket_logging.transition_fastly_logs
+  id = "govuk-production-transition-fastly-logs"
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.transition_fastly_logs
+  id = "govuk-production-transition-fastly-logs"
+}
+
+import {
+  to = aws_iam_user.logs_writer
+  id = "govuk-production-fastly-logs-writer"
+}
+
+import {
+  to = aws_iam_policy.logs_writer
+  id = "arn:aws:iam::172025368201:policy/fastly-logs-production-logs-writer-policy"
+}
+
+import {
+  to = aws_glue_catalog_database.fastly_logs
+  id = "172025368201:fastly_logs"
+}
+
+import {
+  to = aws_iam_role_policy_attachment.aws-glue-service-role-service-attachment
+  id = "AWSGlueServiceRole-fastly-logs/arn:aws:iam::aws:policy/service-role/AWSGlueServiceRole"
+}
+
+import {
+  to = aws_iam_role.glue
+  id = "AWSGlueServiceRole-fastly-logs"
+}
+
+import {
+  to = aws_iam_role_policy.fastly_logs_policy
+  id = "AWSGlueServiceRole-fastly-logs:govuk-production-fastly-logs-glue-policy"
+}
+
+import {
+  to = aws_glue_crawler.govuk_www
+  id = "GOV.UK fastly logs"
+}
+
+import {
+  to = aws_glue_catalog_table.govuk_www
+  id = "172025368201:fastly_logs:govuk_www"
+}
+
+import {
+  to = aws_glue_crawler.govuk_assets
+  id = "Assets fastly logs"
+}
+
+import {
+  id = "172025368201:fastly_logs:govuk_assets"
+  to = aws_glue_catalog_table.govuk_assets
+}
+
+import {
+  to = aws_glue_crawler.bouncer
+  id = "Bouncer fastly logs"
+}
+
+import {
+  id = "172025368201:fastly_logs:bouncer"
+  to = aws_glue_catalog_table.bouncer
+}
+
+import {
+  to = aws_iam_user.transition_downloader
+  id = "govuk-production-transition-downloader"
+}
+
+import {
+  to = aws_iam_policy.transition_downloader
+  id = "arn:aws:iam::172025368201:policy/fastly-logs-production-transition-downloader-policy"
+}
+
+import {
+  to = aws_iam_role.transition_executor
+  id = "AWSLambdaRole-transition-executor"
+}
+
+import {
+  to = aws_iam_policy.transition_executor
+  id = "arn:aws:iam::172025368201:policy/fastly-logs-production-transition-executor-policy"
+}
+
+import {
+  to = aws_iam_role_policy_attachment.transition_executor
+  id = "AWSLambdaRole-transition-executor/arn:aws:iam::172025368201:policy/fastly-logs-production-transition-executor-policy"
+}
+
+import {
+  to = aws_athena_named_query.transition_logs
+  id = "b475cbb1-0212-4192-a6db-de83b400ee00"
+}
+
+import {
+  to = aws_lambda_function.transition_executor
+  id = "govuk-production-transition"
+}
+
+import {
+  to = aws_cloudwatch_event_rule.transition_executor_daily
+  id = "transition_executor_daily"
+}
+
+import {
+  to = aws_cloudwatch_event_target.transition_executor_daily
+  id = "transition_executor_daily/terraform-20180907135027830200000002"
+}
+
+import {
+  to = aws_lambda_permission.cloudwatch_transition_executor_daily_permission
+  id = "govuk-production-transition/AllowExecutionFromCloudWatch"
+}
diff --git a/logs/production.govuk.backend b/logs/production.govuk.backend
new file mode 100644
index 0000000..418223c
--- /dev/null
+++ b/logs/production.govuk.backend
@@ -0,0 +1,4 @@
+bucket  = "govuk-terraform-steppingstone-production"
+key     = "govuk/infra-fastly-logs.tfstate"
+encrypt = true
+region  = "eu-west-1"
diff --git a/logs/staging.govuk.backend b/logs/staging.govuk.backend
new file mode 100644
index 0000000..ea9a212
--- /dev/null
+++ b/logs/staging.govuk.backend
@@ -0,0 +1,4 @@
+bucket  = "govuk-terraform-steppingstone-staging"
+key     = "govuk/infra-fastly-logs.tfstate"
+encrypt = true
+region  = "eu-west-1"
diff --git a/logs/transition_logs.py b/logs/transition_logs.py
new file mode 100644
index 0000000..ba5f3de
--- /dev/null
+++ b/logs/transition_logs.py
@@ -0,0 +1,40 @@
+from datetime import date, timedelta
+import os
+
+import boto3
+
+
+athena = boto3.client('athena')
+
+
+def get_query_string():
+    named_query_id = os.environ['NAMED_QUERY_ID']
+    response = athena.get_named_query(NamedQueryId=named_query_id)
+    return response['NamedQuery']['QueryString']
+
+
+def generate_output_location():
+    bucket_name = os.environ['BUCKET_NAME']
+    yesterday = date.today() - timedelta(days=1)
+    filename = f'athena_{yesterday.isoformat()}'
+    return f's3://{bucket_name}/{filename}'
+
+
+def execute_query(query_string):
+    database_name = os.environ['DATABASE_NAME']
+    output_location = generate_output_location()
+
+    athena.start_query_execution(
+        QueryString=query_string,
+        QueryExecutionContext={
+            'Database': database_name,
+        },
+        ResultConfiguration={
+            'OutputLocation': output_location,
+        },
+    )
+
+
+def lambda_handler(*args, **kwargs):
+    query_string = get_query_string()
+    execute_query(query_string)
diff --git a/logs/variables.tf b/logs/variables.tf
new file mode 100644
index 0000000..236d0d9
--- /dev/null
+++ b/logs/variables.tf
@@ -0,0 +1,10 @@
+variable "environment" {
+  type = string
+  default = "production"
+}
+
+variable "aws_region" {
+  type        = string
+  description = "AWS region"
+  default     = "eu-west-1"
+}