From 440b0e8960c1e9282497e1739129625c764ce049 Mon Sep 17 00:00:00 2001 From: Mat Moore Date: Wed, 8 May 2024 16:55:22 +0100 Subject: [PATCH] Remove data-platform environment We are no longer using any of this. It also came with a customisation to the standard github actions workflow which we wanted to remove. https://github.com/ministryofjustice/data-catalogue/issues/60 --- .github/CODEOWNERS | 1 - .github/dependabot.yml | 6 +- .github/workflows/data-platform.yml | 66 -- .../reusable_terraform_plan_apply_test.yml | 328 -------- .../environments/data-platform/README.md | 76 -- terraform/environments/data-platform/api.tf | 495 ------------ .../application_variables.auto.tfvars.json | 98 --- .../environments/data-platform/athena.tf | 22 - .../environments/data-platform/cloudtrail.tf | 58 -- .../moj_data_product_metadata_spec.json | 109 --- .../moj_data_product_metadata_spec.json | 114 --- .../v1.0.0/moj_data_product_table_spec.json | 161 ---- .../environment-configurations.tf | 21 - terraform/environments/data-platform/glue.tf | 190 ----- .../data-platform/iam-policies.tf | 25 - .../environments/data-platform/iam-roles.tf | 18 - terraform/environments/data-platform/iam.tf | 731 ------------------ .../data-platform/infra_route53.tf | 277 ------- .../environments/data-platform/lambda.tf | 492 ------------ .../environments/data-platform/locals.tf | 62 -- .../data-platform/networking.auto.tfvars.json | 9 - .../data-platform/observability-platform.tf | 9 - .../environments/data-platform/output.tf | 8 - .../data-platform/platform_backend.tf | 14 - .../data-platform/platform_base_variables.tf | 11 - .../data-platform/platform_data.tf | 173 ----- .../data-platform/platform_locals.tf | 38 - .../data-platform/platform_providers.tf | 58 -- .../data-platform/platform_secrets.tf | 17 - .../data-platform/platform_versions.tf | 13 - terraform/environments/data-platform/s3.tf | 276 ------- .../environments/data-platform/secrets.tf | 16 - .../data-platform/tests/delete_table.py | 37 - .../data-platform/tests/requirements.txt | 2 - .../data-platform/tests/run_smoke_tests.py | 164 ---- .../data-platform/tests/test_data.csv | 21 - .../environments/data-platform/triggers.tf | 42 - .../environments/data-platform/variables.tf | 63 -- terraform/environments/data-platform/waf.tf | 66 -- 39 files changed, 1 insertion(+), 4386 deletions(-) delete mode 100644 .github/workflows/data-platform.yml delete mode 100644 .github/workflows/reusable_terraform_plan_apply_test.yml delete mode 100644 terraform/environments/data-platform/README.md delete mode 100644 terraform/environments/data-platform/api.tf delete mode 100644 terraform/environments/data-platform/application_variables.auto.tfvars.json delete mode 100644 terraform/environments/data-platform/athena.tf delete mode 100644 terraform/environments/data-platform/cloudtrail.tf delete mode 100644 terraform/environments/data-platform/data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json delete mode 100644 terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json delete mode 100644 terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json delete mode 100644 terraform/environments/data-platform/environment-configurations.tf delete mode 100644 terraform/environments/data-platform/glue.tf delete mode 100644 terraform/environments/data-platform/iam-policies.tf delete mode 100644 terraform/environments/data-platform/iam-roles.tf delete mode 100644 terraform/environments/data-platform/iam.tf delete mode 100644 terraform/environments/data-platform/infra_route53.tf delete mode 100644 terraform/environments/data-platform/lambda.tf delete mode 100644 terraform/environments/data-platform/locals.tf delete mode 100644 terraform/environments/data-platform/networking.auto.tfvars.json delete mode 100644 terraform/environments/data-platform/observability-platform.tf delete mode 100644 terraform/environments/data-platform/output.tf delete mode 100644 terraform/environments/data-platform/platform_backend.tf delete mode 100644 terraform/environments/data-platform/platform_base_variables.tf delete mode 100644 terraform/environments/data-platform/platform_data.tf delete mode 100644 terraform/environments/data-platform/platform_locals.tf delete mode 100644 terraform/environments/data-platform/platform_providers.tf delete mode 100644 terraform/environments/data-platform/platform_secrets.tf delete mode 100644 terraform/environments/data-platform/platform_versions.tf delete mode 100644 terraform/environments/data-platform/s3.tf delete mode 100644 terraform/environments/data-platform/secrets.tf delete mode 100644 terraform/environments/data-platform/tests/delete_table.py delete mode 100644 terraform/environments/data-platform/tests/requirements.txt delete mode 100644 terraform/environments/data-platform/tests/run_smoke_tests.py delete mode 100644 terraform/environments/data-platform/tests/test_data.csv delete mode 100644 terraform/environments/data-platform/triggers.tf delete mode 100644 terraform/environments/data-platform/variables.tf delete mode 100644 terraform/environments/data-platform/waf.tf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3201411c014..6dea7d956fb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,7 +18,6 @@ /terraform/environments/data-and-insights-wepi @ministryofjustice/data-and-insights-hub @ministryofjustice/modernisation-platform /terraform/environments/data-platform-apps-and-tools @ministryofjustice/data-platform-apps-and-tools @ministryofjustice/modernisation-platform /terraform/environments/data-platform-compute @ministryofjustice/data-platform-apps-and-tools @ministryofjustice/modernisation-platform -/terraform/environments/data-platform @ministryofjustice/data-platform-apps-and-tools @ministryofjustice/data-platform-labs @ministryofjustice/modernisation-platform /terraform/environments/delius-core @ministryofjustice/hmpps-migration @ministryofjustice/modernisation-platform /terraform/environments/delius-iaps @ministryofjustice/hmpps-dba @ministryofjustice/hmpps-migration @ministryofjustice/modernisation-platform /terraform/environments/delius-jitbit @ministryofjustice/hmpps-delius-jitbit-devs @ministryofjustice/hmpps-migration @ministryofjustice/modernisation-platform diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f2e41fd417c..d96052af91a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,4 +1,4 @@ -# This file is auto-generated here, do not manually amend. +# This file is auto-generated here, do not manually amend. # scripts/generate-dependabot.sh version: 2 @@ -105,10 +105,6 @@ updates: directory: "/terraform/environments/data-and-insights-wepi" schedule: interval: "daily" - - package-ecosystem: "terraform" - directory: "/terraform/environments/data-platform" - schedule: - interval: "daily" - package-ecosystem: "terraform" directory: "/terraform/environments/data-platform-apps-and-tools" schedule: diff --git a/.github/workflows/data-platform.yml b/.github/workflows/data-platform.yml deleted file mode 100644 index 2ad087f0055..00000000000 --- a/.github/workflows/data-platform.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -name: data-platform -on: - push: - branches: - - main - paths: - - "terraform/environments/data-platform/**" - - ".github/workflows/data-platform.yml" - - pull_request: - branches: - - main - types: [opened, edited, reopened, synchronize] - paths: - - "terraform/environments/data-platform/**" - - ".github/workflows/data-platform.yml" - - workflow_dispatch: - inputs: - action: - description: "Set either [deploy|destroy]." - default: "deploy" - required: true - type: string - options: - - deploy - - destroy - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - strategy: - uses: ./.github/workflows/reusable_terraform_strategy.yml - if: inputs.action != 'destroy' - with: - application: "${{ github.workflow }}" - - terraform: - needs: strategy - if: inputs.action != 'destroy' - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.strategy.outputs.matrix) }} - uses: ./.github/workflows/reusable_terraform_plan_apply_test.yml - with: - application: "${{ github.workflow }}" - environment: "${{ matrix.target }}" - action: "${{ matrix.action }}" - secrets: - modernisation_platform_environments: "${{ secrets.MODERNISATION_PLATFORM_ENVIRONMENTS }}" - pipeline_github_token: "${{ secrets.MODERNISATION_PLATFORM_CI_USER_ENVIRONMENTS_REPO_PAT }}" - - destroy-development: - if: inputs.action == 'destroy' - uses: ./.github/workflows/reusable_terraform_plan_apply.yml - with: - application: "${{ github.workflow }}" - environment: "development" - action: "plan_apply" - plan_apply_tfargs: "-destroy" - secrets: - modernisation_platform_environments: "${{ secrets.MODERNISATION_PLATFORM_ENVIRONMENTS }}" - pipeline_github_token: "${{ secrets.MODERNISATION_PLATFORM_CI_USER_ENVIRONMENTS_REPO_PAT }}" diff --git a/.github/workflows/reusable_terraform_plan_apply_test.yml b/.github/workflows/reusable_terraform_plan_apply_test.yml deleted file mode 100644 index 383ef3285b8..00000000000 --- a/.github/workflows/reusable_terraform_plan_apply_test.yml +++ /dev/null @@ -1,328 +0,0 @@ ---- -name: terraform plan apply - -# Reusable pipeline for running terraform plan and/or apply on a single -# modernisation platform application environment, e.g. nomis-test. -# -# Constraints: -# - The terraform state must be in a workspace with the same name as the -# application account, e.g. ${application}-${environment} -# -# Features: -# - redacts plan and apply output for use in public repo -# - the apply step sets a deployment environment so a separate approval -# step can be added in the github UI. -# - the apply step is skipped if there is nothing to do in the plan (to -# avoid unnecessary apply approvals) -# - you can optional refresh state as part of the plan (useful if there -# are often AWS changes outside of terraform, and you don't want to -# see this in the plan step) -# - you can optionally post PR plans into the corresponding PR -# - colour output is used unless a PR plan is going to be posted into -# a PR. - -on: - workflow_call: - inputs: - application: - type: string - required: true - description: "Name of the application, e.g. nomis" - environment: - type: string - required: true - description: "Name of the environment, e.g. development" - action: - type: string - required: false - description: "Set to plan or plan_apply" - default: plan - terraform_version: - type: string - required: false - description: "The terraform version to use" - default: "~1" - plan_apply_tfargs: - type: string - required: false - description: "Any additional terraform arguments to be passed in to terraform plan/apply, e.g. -var 'foo=bar'" - default: "" - do_state_refresh_on_plan: - type: boolean - required: false - description: "Set to true to do a state refresh prior to the plan" - default: false - post_plan_to_pr: - type: boolean - required: false - description: "Set to true to post terraform plan as a comment to the PR" - default: false - secrets: - modernisation_platform_environments: - required: true - pipeline_github_token: - required: true - -env: - ACCOUNT_NAME: "${{ inputs.application }}-${{ inputs.environment }}" - WORKSPACE_NAME: "${{ inputs.application }}-${{ inputs.environment }}" - ENVIRONMENT_MANAGEMENT: "${{ secrets.modernisation_platform_environments }}" - GITHUB_TOKEN: "${{ secrets.pipeline_github_token }}" - -jobs: - plan: - name: "plan" - runs-on: ubuntu-latest - outputs: - plan_exitcode: "${{ steps.plan.outputs.exitcode }}" - steps: - - name: Debug - run: | - echo "application=${{ inputs.application }}" - echo "environment=${{ inputs.environment }}" - echo "action=${{ inputs.action }}" - echo "init_plan_apply_tfargs=${{ inputs.init_plan_apply_tfargs }}" - echo "plan_apply_tfargs=${{ inputs.plan_apply_tfargs }}" - echo "do_state_refresh_on_plan=${{ inputs.do_state_refresh_on_plan }}" - echo "post_plan_to_pr=${{ inputs.post_plan_to_pr }}" - - name: Checkout Repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - - name: Get AWS Account Number - run: | - ACCOUNT_NUMBER=$(jq -r -e --arg account_name "${ACCOUNT_NAME}" '.account_ids[$account_name]' <<< $ENVIRONMENT_MANAGEMENT) - echo "ACCOUNT_NUMBER=${ACCOUNT_NUMBER}" >> $GITHUB_ENV - - - name: Get Backend AWS Account Number - run: | - BACKEND_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT) - echo "BACKEND_NUMBER=${BACKEND_NUMBER}" >> $GITHUB_ENV - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" - role-session-name: githubactionsrolesession - aws-region: "eu-west-2" - - - name: Setup Terraform - uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1 - with: - terraform_version: "${{ inputs.terraform_version }}" - terraform_wrapper: false - - - name: Terraform Init - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - terraform --version - echo "terraform init -backend-config=assume_role={role_arn=\"arn:aws:iam::${{env.BACKEND_NUMBER}}:role/modernisation-account-terraform-state-member-access\"}" - terraform init -backend-config=assume_role={role_arn=\"arn:aws:iam::${{env.BACKEND_NUMBER}}:role/modernisation-account-terraform-state-member-access\"} - - - name: Terraform Workspace Select - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - terraform workspace select "${WORKSPACE_NAME}" - - - name: Terraform State Refresh (Optional) - if: inputs.do_state_refresh_on_plan == true - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - set -o pipefail - tf_args="${{ inputs.init_plan_apply_tfargs }} ${{ inputs.plan_apply_tfargs }}" - echo "terraform apply -refresh-only -auto-approve ${tf_args}" - terraform apply -refresh-only -auto-approve ${tf_args} | bash ${GITHUB_WORKSPACE}/scripts/redact-output.sh - - - name: Terraform Plan - id: plan - env: - POST_PLAN_TO_PR: "${{ github.event_name == 'pull_request' && inputs.post_plan_to_pr == true }}" - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - set -o pipefail - exitcode=0 - tf_args="-detailed-exitcode ${{ inputs.init_plan_apply_tfargs }} ${{ inputs.plan_apply_tfargs }}" - [[ ${POST_PLAN_TO_PR} == 'true' ]] && tf_args="${tf_args} -no-color" - [[ ${{ inputs.do_state_refresh_on_plan }} == 'true' ]] && tf_args="${tf_args} -refresh=false" - echo "terraform plan ${tf_args}" - terraform plan ${tf_args} | bash ${GITHUB_WORKSPACE}/scripts/redact-output.sh | tee tfplan.txt || exitcode=$? - echo "exitcode=${exitcode}" # 0=clean plan, 1=error, 2=stuff in plan - echo "exitcode=${exitcode}" >> $GITHUB_OUTPUT - (( exitcode == 1 )) && exit 1 || exit 0 - - - name: Create Plan PR message (Optional) - if: github.event_name == 'pull_request' && steps.plan.outputs.exitcode == '2' && inputs.post_plan_to_pr == true - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - comment() { - url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - len=$(cat tfplan.txt | wc -c) - echo '**`${{ env.WORKSPACE_NAME }}`** terraform plan on `${{ github.event_name }}` event [#${{ github.run_number }}]('${url}')' - echo - echo '```' - head -c 65476 tfplan.txt | sed -n '/Terraform will perform/,$p' - echo - echo '```' - if [[ $len -gt 65476 ]]; then - echo "** Truncated output. See $url for the rest **" - fi - } - echo 'TF_PLAN_OUT<> $GITHUB_ENV - comment >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - - name: Hide Previous PR comment (Optional) - if: ${{ github.event_name == 'pull_request' }} - working-directory: "scripts/minimise-comments" - env: - COMMENT_BODY_CONTAINS: "**`${{ env.WORKSPACE_NAME }}`**" - PR_NUMBER: "${{ github.event.pull_request.number }}" - run: | - go build - ./minimise-comments - - - name: Post Plan to PR (Optional) - if: github.event_name == 'pull_request' && steps.plan.outputs.exitcode == '2' && inputs.post_plan_to_pr == true - env: - message: "${{ env.TF_PLAN_OUT }}" - run: | - escaped_message=$(echo "$message" | jq -Rsa .) - curl -sS -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ env.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments" \ - -d '{"body":'"${escaped_message}"'}' - - terratest: - name: "terratest" - needs: plan - if: inputs.action == 'plan_apply' && needs.plan.outputs.plan_exitcode == '2' && inputs.environment == 'development' - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - - name: Get AWS Account Number - run: | - ACCOUNT_NUMBER=$(jq -r -e --arg account_name "${ACCOUNT_NAME}" '.account_ids[$account_name]' <<< $ENVIRONMENT_MANAGEMENT) - echo "ACCOUNT_NUMBER=${ACCOUNT_NUMBER}" >> $GITHUB_ENV - - - name: Get Backend AWS Account Number - run: | - BACKEND_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT) - echo "BACKEND_NUMBER=${BACKEND_NUMBER}" >> $GITHUB_ENV - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" - role-session-name: githubactionsrolesession - aws-region: "eu-west-2" - - # - name: Run Terratest - # uses: cloudposse/github-action-terratest@main - # with: - # sourceDir: test/src - - apply: - name: "apply" - needs: plan - if: inputs.action == 'plan_apply' && needs.plan.outputs.plan_exitcode == '2' - runs-on: ubuntu-latest - environment: "${{ inputs.application }}-${{ inputs.environment }}" - steps: - - name: Checkout Repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - - name: Get AWS Account Number - run: | - ACCOUNT_NUMBER=$(jq -r -e --arg account_name "${ACCOUNT_NAME}" '.account_ids[$account_name]' <<< $ENVIRONMENT_MANAGEMENT) - echo "ACCOUNT_NUMBER=${ACCOUNT_NUMBER}" >> $GITHUB_ENV - - - name: Get Backend AWS Account Number - run: | - BACKEND_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT) - echo "BACKEND_NUMBER=${BACKEND_NUMBER}" >> $GITHUB_ENV - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" - role-session-name: githubactionsrolesession - aws-region: "eu-west-2" - - - name: Setup Terraform - uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1 - with: - terraform_version: "${{ inputs.terraform_version }}" - terraform_wrapper: false - - - name: Terraform Init - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - terraform --version - echo "terraform init -backend-config=assume_role={role_arn=\"arn:aws:iam::${{env.BACKEND_NUMBER}}:role/modernisation-account-terraform-state-member-access\"}" - terraform init -backend-config=assume_role={role_arn=\"arn:aws:iam::${{env.BACKEND_NUMBER}}:role/modernisation-account-terraform-state-member-access\"} - - - name: Terraform Workspace Select - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - terraform workspace select "${WORKSPACE_NAME}" - - - name: Terraform Plan - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - set -o pipefail - tf_args="-out x.tfplan ${{ inputs.init_plan_apply_tfargs }} ${{ inputs.plan_apply_tfargs }}" - echo "terraform plan ${tf_args}" - terraform plan ${tf_args} | bash ${GITHUB_WORKSPACE}/scripts/redact-output.sh - - - name: Terraform Apply - working-directory: "terraform/environments/${{ inputs.application }}" - run: | - set -o pipefail - tf_args="${{ inputs.init_plan_apply_tfargs }} ${{ inputs.plan_apply_tfargs }} x.tfplan" - echo "terraform apply ${tf_args}" - terraform apply ${tf_args} | bash ${GITHUB_WORKSPACE}/scripts/redact-output.sh - - smoketest: - name: "smoketest" - needs: apply - if: inputs.action == 'plan_apply' && needs.plan.outputs.plan_exitcode == '2' && inputs.environment == 'development' - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4 - - - name: Get AWS Account Number - run: | - ACCOUNT_NUMBER=$(jq -r -e --arg account_name "${ACCOUNT_NAME}" '.account_ids[$account_name]' <<< $ENVIRONMENT_MANAGEMENT) - echo "ACCOUNT_NUMBER=${ACCOUNT_NUMBER}" >> $GITHUB_ENV - - - name: Get Backend AWS Account Number - run: | - BACKEND_NUMBER=$(jq -r -e '.modernisation_platform_account_id' <<< $ENVIRONMENT_MANAGEMENT) - echo "BACKEND_NUMBER=${BACKEND_NUMBER}" >> $GITHUB_ENV - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: "arn:aws:iam::${{ env.ACCOUNT_NUMBER }}:role/github-actions" - role-session-name: githubactionsrolesession - aws-region: "eu-west-2" - - - name: Setup python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Run smoke tests - working-directory: ./terraform/environments/data-platform/tests/ - run: | - export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ - $(aws sts assume-role \ - --role-arn arn:aws:iam::${ACCOUNT_NUMBER}:role/MemberInfrastructureAccess \ - --role-session-name smoketesting-data-platform \ - --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ - --output text)) - export API_AUTH=$(aws secretsmanager get-secret-value --secret-id data-platform-api-auth-token --query 'SecretString' --output text --no-cli-pager) - pip install -r requirements.txt && python run_smoke_tests.py diff --git a/terraform/environments/data-platform/README.md b/terraform/environments/data-platform/README.md deleted file mode 100644 index 9aa2658704c..00000000000 --- a/terraform/environments/data-platform/README.md +++ /dev/null @@ -1,76 +0,0 @@ -# Service Runbook - - - -_If you have any questions surrounding this page please post in the `#team-name` channel._ - -## Mandatory Information - -### **Last review date:** - - - -### **Description:** - - - -### **Service URLs:** - - - -### **Incident response hours:** - - - -### **Incident contact details:** - - - -### **Service team contact:** - - - -### **Hosting environment:** - -Modernisation Platform - - - -## Optional - -### **Other URLs:** - - - -### **Expected speed and frequency of releases:** - - - -### **Automatic alerts:** - - - -### **Impact of an outage:** - - - -### **Out of hours response types:** - - - -### **Consumers of this service:** - - - -### **Services consumed by this:** - - - -### **Restrictions on access:** - - - -### **How to resolve specific issues:** - - diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf deleted file mode 100644 index dacf3283f1c..00000000000 --- a/terraform/environments/data-platform/api.tf +++ /dev/null @@ -1,495 +0,0 @@ - -resource "aws_api_gateway_rest_api" "data_platform" { - name = "data_platform" -} - -resource "aws_api_gateway_deployment" "deployment" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - triggers = { - # NOTE: The configuration below will satisfy ordering considerations, - # but not pick up all future REST API changes. More advanced patterns - # are possible, such as using the filesha1() function against the - # Terraform configuration file(s) or removing the .id references to - # calculate a hash against whole resources. Be aware that using whole - # resources will show a difference after the initial implementation. - # It will stabilize to only change when resources change afterwards. - redeployment = sha1(jsonencode([ - aws_api_gateway_resource.docs, - aws_api_gateway_resource.data_product, - aws_api_gateway_resource.register_data_product, - aws_api_gateway_resource.data_product_name, - aws_api_gateway_resource.data_product_preview, - aws_api_gateway_resource.data_product_table, - aws_api_gateway_resource.data_product_table_name, - aws_api_gateway_resource.upload_data_for_data_product_table_name, - aws_api_gateway_resource.schema_for_data_product_table_name, - aws_api_gateway_method.preview_data_from_data_product, - aws_api_gateway_method.docs, - aws_api_gateway_method.register_data_product, - aws_api_gateway_method.upload_data_for_data_product_table_name, - aws_api_gateway_method.create_schema_for_data_product_table_name, - aws_api_gateway_method.get_schema_for_data_product_table_name, - aws_api_gateway_method.update_data_product, - aws_api_gateway_method.update_schema_for_data_product_table_name, - aws_api_gateway_method.delete_table_for_data_product, - aws_api_gateway_method.delete_data_product, - aws_api_gateway_integration.docs_to_lambda, - aws_api_gateway_integration.upload_data_for_data_product_table_name_to_lambda, - aws_api_gateway_integration.proxy_to_lambda, - aws_api_gateway_integration.docs_lambda_root, - aws_api_gateway_integration.register_data_product_to_lambda, - aws_api_gateway_integration.create_schema_for_data_product_table_name_to_lambda, - aws_api_gateway_integration.get_schema_for_data_product_table_name_to_lambda, - aws_api_gateway_integration.update_data_product_to_lambda, - aws_api_gateway_integration.update_schema_for_data_product_table_name_to_lambda, - aws_api_gateway_integration.preview_data_from_data_product_lambda, - aws_api_gateway_integration.delete_table_for_data_product_to_lambda, - aws_api_gateway_integration.delete_data_product_lambda, - ])) - } - - lifecycle { - create_before_destroy = true - } -} - -resource "aws_api_gateway_stage" "default_stage" { - deployment_id = aws_api_gateway_deployment.deployment.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - stage_name = local.environment - - depends_on = [aws_api_gateway_account.api_gateway_account] - - access_log_settings { - destination_arn = aws_cloudwatch_log_group.data_platform_api.arn - format = jsonencode({ - requestId = "$context.requestId" - requestTime = "$context.requestTime" - requestTimeEpoch = "$context.requestTimeEpoch" - ip = "$context.identity.sourceIp" - caller = "$context.identity.caller" - user = "$context.identity.user" - path = "$context.path" - resourcePath = "$context.resourcePath" - method = "$context.httpMethod" - status = "$context.status" - protocol = "$context.protocol" - responseLength = "$context.responseLength" - }) - } -} - -resource "aws_cloudwatch_log_group" "data_platform_api" { - name = "data_platform_api" -} - -resource "aws_api_gateway_account" "api_gateway_account" { - cloudwatch_role_arn = aws_iam_role.api_gateway_cloud_watch_role.arn -} - -resource "aws_api_gateway_method_settings" "api_gateway_log_settings" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - stage_name = local.environment - method_path = "*/*" - - settings { - logging_level = "INFO" - } -} - -resource "aws_api_gateway_authorizer" "authorizer" { - name = "authorizer-${local.environment}" - rest_api_id = aws_api_gateway_rest_api.data_platform.id - authorizer_uri = module.data_product_authorizer_lambda.lambda_function_invoke_arn - authorizer_credentials = aws_iam_role.authoriser_role.arn - identity_source = "method.request.header.authorizationToken" -} - -# /data-product resource -resource "aws_api_gateway_resource" "data_product" { - parent_id = aws_api_gateway_rest_api.data_platform.root_resource_id - path_part = "data-product" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/register resource -resource "aws_api_gateway_resource" "register_data_product" { - parent_id = aws_api_gateway_resource.data_product.id - path_part = "register" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/register POST method -resource "aws_api_gateway_method" "register_data_product" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "POST" - resource_id = aws_api_gateway_resource.register_data_product.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true - } -} - -# /data-product/register lambda integration -resource "aws_api_gateway_integration" "register_data_product_to_lambda" { - http_method = aws_api_gateway_method.register_data_product.http_method - resource_id = aws_api_gateway_resource.register_data_product.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_create_metadata_lambda.lambda_function_invoke_arn -} - -# /data-product/{data-product-name} resource -resource "aws_api_gateway_resource" "data_product_name" { - parent_id = aws_api_gateway_resource.data_product.id - path_part = "{data-product-name}" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/{data-product} PUT method -resource "aws_api_gateway_method" "update_data_product" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "PUT" - resource_id = aws_api_gateway_resource.data_product_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - } -} - -# PUT /data-product/{data-product-name} lambda integration -resource "aws_api_gateway_integration" "update_data_product_to_lambda" { - http_method = aws_api_gateway_method.update_data_product.http_method - resource_id = aws_api_gateway_resource.data_product_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_update_metadata_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name" - } -} - - -# /data-product/{data-product-name}/table resource -resource "aws_api_gateway_resource" "data_product_table" { - parent_id = aws_api_gateway_resource.data_product_name.id - path_part = "table" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/{data-product-name}/table/{table-name} resource -resource "aws_api_gateway_resource" "data_product_table_name" { - parent_id = aws_api_gateway_resource.data_product_table.id - path_part = "{table-name}" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/{data-product-name}/table/{table-name}/upload resource -resource "aws_api_gateway_resource" "upload_data_for_data_product_table_name" { - parent_id = aws_api_gateway_resource.data_product_table_name.id - path_part = "upload" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/{data-product-name}/table/{table-name}/schema resource -resource "aws_api_gateway_resource" "schema_for_data_product_table_name" { - parent_id = aws_api_gateway_resource.data_product_table_name.id - path_part = "schema" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - -# /data-product/{data-product-name}/table/{table-name}/upload POST method -resource "aws_api_gateway_method" "upload_data_for_data_product_table_name" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "POST" - resource_id = aws_api_gateway_resource.upload_data_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name}/upload lambda integration -resource "aws_api_gateway_integration" "upload_data_for_data_product_table_name_to_lambda" { - http_method = aws_api_gateway_method.upload_data_for_data_product_table_name.http_method - resource_id = aws_api_gateway_resource.upload_data_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_presigned_url_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema POST method -resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "POST" - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema lambda integration -resource "aws_api_gateway_integration" "create_schema_for_data_product_table_name_to_lambda" { - http_method = aws_api_gateway_method.create_schema_for_data_product_table_name.http_method - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_create_schema_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema GET method -resource "aws_api_gateway_method" "get_schema_for_data_product_table_name" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "GET" - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema lambda integration -resource "aws_api_gateway_integration" "get_schema_for_data_product_table_name_to_lambda" { - http_method = aws_api_gateway_method.get_schema_for_data_product_table_name.http_method - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.get_schema_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema PUT method -resource "aws_api_gateway_method" "update_schema_for_data_product_table_name" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "PUT" - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name}/schema lambda integration -resource "aws_api_gateway_integration" "update_schema_for_data_product_table_name_to_lambda" { - http_method = aws_api_gateway_method.update_schema_for_data_product_table_name.http_method - resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_update_schema_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# /data-product/{data-product-name}/table/{table-name} DELETE method -resource "aws_api_gateway_method" "delete_table_for_data_product" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "DELETE" - resource_id = aws_api_gateway_resource.data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name} (delete table and data) lambda integration -resource "aws_api_gateway_integration" "delete_table_for_data_product_to_lambda" { - http_method = aws_api_gateway_method.delete_table_for_data_product.http_method - resource_id = aws_api_gateway_resource.data_product_table_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.delete_table_for_data_product_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# API docs endpoint - -resource "aws_api_gateway_resource" "docs" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - parent_id = aws_api_gateway_rest_api.data_platform.root_resource_id - path_part = "docs" -} - -resource "aws_api_gateway_method" "docs" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_resource.docs.id - http_method = "ANY" - authorization = "NONE" -} - -resource "aws_api_gateway_integration" "docs_to_lambda" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_method.docs.resource_id - http_method = aws_api_gateway_method.docs.http_method - - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_docs_lambda.lambda_function_invoke_arn -} - -resource "aws_api_gateway_resource" "proxy" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - parent_id = aws_api_gateway_resource.docs.id - path_part = "{proxy+}" -} - -resource "aws_api_gateway_method" "proxy" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_resource.proxy.id - http_method = "ANY" - authorization = "NONE" -} - -resource "aws_api_gateway_integration" "proxy_to_lambda" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_method.proxy.resource_id - http_method = aws_api_gateway_method.proxy.http_method - - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.data_product_docs_lambda.lambda_function_invoke_arn -} - -resource "aws_api_gateway_method" "proxy_root" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_rest_api.data_platform.root_resource_id - http_method = "ANY" - authorization = "NONE" -} - -resource "aws_api_gateway_integration" "docs_lambda_root" { - rest_api_id = aws_api_gateway_rest_api.data_platform.id - resource_id = aws_api_gateway_method.proxy_root.resource_id - http_method = aws_api_gateway_method.proxy_root.http_method - - integration_http_method = "POST" - type = "MOCK" - uri = module.data_product_docs_lambda.lambda_function_invoke_arn - - lifecycle { - ignore_changes = all - } -} - -# Preview data - -# /data-product/{data-product-name}/table/{table-name}/preview resource -resource "aws_api_gateway_resource" "data_product_preview" { - parent_id = aws_api_gateway_resource.data_product_table_name.id - path_part = "preview" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - - -# /data-product/{data-product-name}/table/{table-name}/preview GET method -resource "aws_api_gateway_method" "preview_data_from_data_product" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "GET" - resource_id = aws_api_gateway_resource.data_product_preview.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - "method.request.path.table-name" = true, - } -} - -# /data-product/{data-product-name}/table/{table-name}/preview lambda integration -resource "aws_api_gateway_integration" "preview_data_from_data_product_lambda" { - http_method = aws_api_gateway_method.preview_data_from_data_product.http_method - resource_id = aws_api_gateway_resource.data_product_preview.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.preview_data_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name", - "integration.request.path.table-name" = "method.request.path.table-name", - } -} - -# /data-product/{data-product-name} DELETE method -resource "aws_api_gateway_method" "delete_data_product" { - authorization = "CUSTOM" - authorizer_id = aws_api_gateway_authorizer.authorizer.id - http_method = "DELETE" - resource_id = aws_api_gateway_resource.data_product_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - - request_parameters = { - "method.request.header.Authorization" = true, - "method.request.path.data-product-name" = true, - } -} - -# /data-product/{data-product-name} DELETE lambda integration -resource "aws_api_gateway_integration" "delete_data_product_lambda" { - http_method = aws_api_gateway_method.delete_data_product.http_method - resource_id = aws_api_gateway_resource.data_product_name.id - rest_api_id = aws_api_gateway_rest_api.data_platform.id - integration_http_method = "POST" - type = "AWS_PROXY" - uri = module.delete_data_product_lambda.lambda_function_invoke_arn - - request_parameters = { - "integration.request.path.data-product-name" = "method.request.path.data-product-name" - } -} diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json deleted file mode 100644 index 50931644648..00000000000 --- a/terraform/environments/data-platform/application_variables.auto.tfvars.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "docs_versions": { - "development": "1.0.14", - "test": "1.0.14", - "preproduction": "1.0.14", - "production": "1.0.14" - }, - "authorizer_versions": { - "development": "1.2.0", - "test": "1.2.0", - "preproduction": "1.2.0", - "production": "1.2.0" - }, - "presigned_url_versions": { - "development": "1.7.0", - "test": "1.7.0", - "preproduction": "1.7.0", - "production": "1.7.0" - }, - "athena_load_versions": { - "development": "3.0.1", - "test": "3.0.1", - "preproduction": "3.0.1", - "production": "3.0.1" - }, - "create_metadata_versions": { - "development": "2.2.0", - "test": "2.2.0", - "preproduction": "2.2.0", - "production": "2.2.0" - }, - "resync_unprocessed_files_versions": { - "development": "2.2.0", - "test": "2.2.0", - "preproduction": "2.2.0", - "production": "2.2.0" - }, - "reload_data_product_versions": { - "development": "2.2.0", - "test": "2.2.0", - "preproduction": "2.2.0", - "production": "2.2.0" - }, - "landing_to_raw_versions": { - "development": "1.5.1", - "test": "1.5.1", - "preproduction": "1.5.1", - "production": "1.5.1" - }, - "create_schema_versions": { - "development": "1.2.2", - "test": "1.2.2", - "preproduction": "1.2.2", - "production": "1.2.2" - }, - "get_schema_versions": { - "development": "1.3.0", - "test": "1.3.0", - "preproduction": "1.3.0", - "production": "1.3.0" - }, - "update_metadata_versions": { - "development": "1.1.0", - "test": "1.1.0", - "preproduction": "1.1.0", - "production": "1.1.0" - }, - "update_schema_versions": { - "development": "1.2.0", - "test": "1.2.0", - "preproduction": "1.2.0", - "production": "1.2.0" - }, - "preview_data_versions": { - "development": "2.2.0", - "test": "2.2.0", - "preproduction": "2.2.0", - "production": "2.2.0" - }, - "delete_table_for_data_product_versions": { - "development": "2.4.0", - "test": "2.4.0", - "preproduction": "2.4.0", - "production": "2.4.0" - }, - "push_to_catalogue_versions": { - "development": "1.1.0", - "test": "1.1.0", - "preproduction": "1.1.0", - "production": "1.1.0" - }, - "delete_data_product_versions": { - "development": "2.2.0", - "test": "2.2.0", - "preproduction": "2.2.0", - "production": "2.2.0" - } -} diff --git a/terraform/environments/data-platform/athena.tf b/terraform/environments/data-platform/athena.tf deleted file mode 100644 index c0134316d41..00000000000 --- a/terraform/environments/data-platform/athena.tf +++ /dev/null @@ -1,22 +0,0 @@ - -resource "aws_athena_workgroup" "data_product_athena_workgroup" { - name = "data_product_workgroup" - - configuration { - enforce_workgroup_configuration = false - publish_cloudwatch_metrics_enabled = true - engine_version { - selected_engine_version = "Athena engine version 3" - } - result_configuration { - output_location = "s3://athena-data-product-query-results-${data.aws_caller_identity.current.account_id}" - - encryption_configuration { - encryption_option = "SSE_S3" - } - acl_configuration { - s3_acl_option = "BUCKET_OWNER_FULL_CONTROL" - } - } - } -} \ No newline at end of file diff --git a/terraform/environments/data-platform/cloudtrail.tf b/terraform/environments/data-platform/cloudtrail.tf deleted file mode 100644 index 2efd27efa12..00000000000 --- a/terraform/environments/data-platform/cloudtrail.tf +++ /dev/null @@ -1,58 +0,0 @@ -# a cloudtrail trail to save log files for putObject S3 events in the landing and data -# buckets -resource "aws_cloudtrail" "data_s3_put_objects" { - name = "data_platform_s3_putobject_trail_${local.environment}" - s3_bucket_name = module.logs_s3_bucket.bucket.id - - # this is needed if monitoring services without a specific region. Don't need for s3 - include_global_service_events = false - - # enabling this would allow detection of modified log files - enable_log_file_validation = false - advanced_event_selector { - name = "Log PutObject events for landing and data S3 buckets" - - field_selector { - field = "eventCategory" - equals = ["Data"] - } - - field_selector { - field = "eventName" - - equals = [ - "PutObject", - "CopyObject", - "CompleteMultipartUpload" - ] - } - - field_selector { - field = "resources.ARN" - - # The trailing slash is intentional; do not exclude it. - starts_with = [ - "${module.data_s3_bucket.bucket.arn}/", - "${module.data_landing_s3_bucket.bucket.arn}/" - ] - } - - # Remove this if we want to log read events too, like getObject - field_selector { - field = "readOnly" - equals = ["false"] - } - - field_selector { - field = "resources.type" - equals = ["AWS::S3::Object"] - } - } - - cloud_watch_logs_group_arn = "${aws_cloudwatch_log_group.data_platform_s3_putobject_trail.arn}:*" # CloudTrail requires the Log Stream wildcard - cloud_watch_logs_role_arn = aws_iam_role.cloudtrail_cloudwatch_role.arn -} - -resource "aws_cloudwatch_log_group" "data_platform_s3_putobject_trail" { - name = "/aws/cloudtrail/data_platform_s3_putobject_trail_${local.environment}" -} diff --git a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json b/terraform/environments/data-platform/data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json deleted file mode 100644 index a1a62773622..00000000000 --- a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.0.0/moj_data_product_metadata_spec.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "title": "Data Product Metadata", - "description": "Specification for MoJ Data Platform Data Product metadata", - "required": [ - "name", - "description", - "domain", - "dataProductOwner", - "dataProductOwnerDisplayName", - "email", - "status", - "dpiaRequired" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the Data Product. Must contain only lowercase letters, numbers, and the underscore character.", - "pattern": "^[a-z0-9_]+$", - "example": "my_data_product" - }, - "description": { - "type": "string", - "description": "Detailed description about what functional area this Data Product is representing, what purpose it has and business related information.", - "example": "this data product hold lots of useful information I want to share with those who may have use for it." - }, - "domain": { - "type": "string", - "description": "The identifier of the domain this Data Product belongs to. Should be one of HQ, HMPPS, OPG, LAA, HMCTS, CICA, or Platforms", - "example": "HMPPS" - }, - "dataProductOwner": { - "type": "string", - "description": "Data Product owner, the unique identifier of the actual user that owns, manages, and receives notifications about the Data Product. To make it technology independent it is usually the email address of the owner.", - "example": "jane.doe@justice.gov.uk" - }, - "dataProductOwnerDisplayName": { - "type": "string", - "description": "The human-readable version of dataProductOwner", - "example": "Jane Doe" - }, - "dataProductMaintainer": { - "type": "string", - "description": "Secondary party who is able to approve DPIA access requests, but who may or may not be legally responsible for the data", - "example": "example.information.asset.owner@justice.gov.uk" - }, - "dataProductMaintainerDisplayName": { - "type": "string", - "description": "The human-readable version of dataProductMaintainer", - "example": "Jonny Data" - }, - "email": { - "type": "string", - "description": "point of contact between consumers and maintainers of the Data Product. It could be the owner or a distribution list, but must be reliable and responsive.", - "example": "jane.doe@justice.gov.uk" - }, - "status": { - "type": "string", - "description": "this is an enum representing the status of this version of the Data Product. Allowed values are: [draft|published|retired]. This is a metadata that communicates the overall status of the Data Product but is not reflected to the actual deployment status.", - "enum": ["draft", "published", "retired"] - }, - "dpiaRequired": { - "type": "boolean", - "description": "Bool for if a data privacy impact assessment (dpia) is required to access this data product", - "example": true - }, - "dpiaLocation": { - "type": "string", - "description": "Data Privacy Impact Assessment (DPIA) file s3 location for this data product. Generated by data platform." - }, - "retentionPeriod": { - "type": "integer", - "description": "Retention period of the data in this data product in days.", - "example": 3650 - }, - "tags": { - "type": "object", - "description": "Additional tags to add.", - "example": { "sandbox": true } - }, - "version": { - "type": "string", - "description": "Data product version of form [major].[minor]. Generated by data platform." - }, - "id": { - "type": "string", - "description": "Data product unique id. Generated by data platform.", - "example": "dp:civil-courts-data:v1.1" - }, - "lastUpdated": { - "type": "string", - "description": "Last data upload date to this data product. Generated by data platform." - }, - "creationDate": { - "type": "string", - "description": "Creation date of the data product. Generated by data platform." - }, - "s3Location": { - "type": "string", - "description": "S3 path to data in this data product. Generated by data platform." - }, - "rowCount": { - "type": "object", - "description": "Total row count of all tables in the data product, as a heuristic. Generated by data platform." - } - }, - "additionalProperties": false -} diff --git a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json b/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json deleted file mode 100644 index 5a3d4f3ec54..00000000000 --- a/terraform/environments/data-platform/data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "title": "Data Product Metadata", - "description": "Specification for MoJ Data Platform Data Product metadata", - "required": [ - "name", - "description", - "domain", - "dataProductOwner", - "dataProductOwnerDisplayName", - "email", - "status", - "dpiaRequired", - "retentionPeriod" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the Data Product. Must contain only lowercase letters, numbers, and the underscore character.", - "pattern": "^[a-z0-9_]+$", - "example": "my_data_product" - }, - "description": { - "type": "string", - "description": "Detailed description about what functional area this Data Product is representing, what purpose it has and business related information.", - "example": "this data product hold lots of useful information I want to share with those who may have use for it." - }, - "domain": { - "type": "string", - "description": "The identifier of the domain this Data Product belongs to. Should be one of HQ, HMPPS, OPG, LAA, HMCTS, CICA, or Platforms", - "example": "HMPPS" - }, - "dataProductOwner": { - "type": "string", - "description": "Data Product owner, the unique identifier of the actual user that owns, manages, and receives notifications about the Data Product. To make it technology independent it is usually the email address of the owner.", - "example": "jane.doe@justice.gov.uk" - }, - "dataProductOwnerDisplayName": { - "type": "string", - "description": "The human-readable version of dataProductOwner", - "example": "Jane Doe" - }, - "dataProductMaintainer": { - "type": "string", - "description": "Secondary party who is able to approve DPIA access requests, but who may or may not be legally responsible for the data", - "example": "example.information.asset.owner@justice.gov.uk" - }, - "dataProductMaintainerDisplayName": { - "type": "string", - "description": "The human-readable version of dataProductMaintainer", - "example": "Jonny Data" - }, - "email": { - "type": "string", - "description": "point of contact between consumers and maintainers of the Data Product. It could be the owner or a distribution list, but must be reliable and responsive.", - "example": "jane.doe@justice.gov.uk" - }, - "status": { - "type": "string", - "description": "this is an enum representing the status of this version of the Data Product. Allowed values are: [draft|published|retired]. This is a metadata that communicates the overall status of the Data Product but is not reflected to the actual deployment status.", - "enum": ["draft", "published", "retired"] - }, - "dpiaRequired": { - "type": "boolean", - "description": "Bool for if a data privacy impact assessment (dpia) is required to access this data product", - "example": true - }, - "dpiaLocation": { - "type": "string", - "description": "Data Privacy Impact Assessment (DPIA) file s3 location for this data product. Generated by data platform." - }, - "retentionPeriod": { - "type": "integer", - "description": "Retention period of the data in this data product in days.", - "example": 3650 - }, - "tags": { - "type": "object", - "description": "Additional tags to add.", - "example": { "sandbox": true } - }, - "version": { - "type": "string", - "description": "Data product version of form [major].[minor]. Generated by data platform." - }, - "id": { - "type": "string", - "description": "Data product unique id. Generated by data platform.", - "example": "dp:civil-courts-data:v1.1" - }, - "lastUpdated": { - "type": "string", - "description": "Last data upload date to this data product. Generated by data platform." - }, - "creationDate": { - "type": "string", - "description": "Creation date of the data product. Generated by data platform." - }, - "s3Location": { - "type": "string", - "description": "S3 path to data in this data product. Generated by data platform." - }, - "rowCount": { - "type": "object", - "description": "Total row count of all tables in the data product, as a heuristic. Generated by data platform." - }, - "schemas": { - "type": "array", - "description": "This contains a list of schemas or tables that are part of the data product. Generated by data platform" - } - }, - "additionalProperties": false -} diff --git a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json b/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json deleted file mode 100644 index 2ac5185f277..00000000000 --- a/terraform/environments/data-platform/data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "title": "Data Product Table Schema", - "description": "Specification for MoJ Data Platform Data Product tables schemas for registering tables in the glue catalog", - "type": "object", - "required": ["tableDescription", "columns"], - "definitions": { - "additional_col_properties": { - "required": ["name", "type", "description"], - "properties": { - "name": { - "type": "string", - "description": "The name of a column within your data.", - "pattern": "^[a-z0-9_]+$", - "maxLength": 128 - }, - "description": { - "type": "string", - "description": "A description of the column that will feed the data catalogue." - } - }, - "additionalProperties": true - }, - "int_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^u?(tiny|small|big|)int$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "bigint" - } - } - } - ] - }, - "number_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^float$|^double$|^decimal\\(\\d{1,2},\\s?\\d{1,2}\\)$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "float" - } - } - } - ] - }, - "string_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^char\\(\\d{1,3}\\)$|^varchar\\(\\d{0,5}\\)$|^varchar$|^string$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "string" - } - } - } - ] - }, - "boolean_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^boolean$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "boolean" - } - } - } - ] - }, - "date_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^date$|^timestamp$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "date" - } - } - } - ] - }, - "struct_types": { - "allOf": [ - { - "$ref": "#/definitions/additional_col_properties" - }, - { - "properties": { - "type": { - "type": "string", - "pattern": "^map_<.+>$|^struct<.+>$", - "description": "The data type of the Column. See https://docs.aws.amazon.com/athena/latest/ug/data-types.html", - "example": "struct" - } - } - } - ] - } - }, - "properties": { - "tableDescription": { - "type": "string", - "description": "A description of the data contained within the table", - "example": "this table contains example data for an example data product." - }, - "columns": { - "type": "array", - "description": "list of objects which relate to columns in your data, each list item will contain, a name of the column, data type of the column and description of the column", - "items": { - "oneOf": [ - { - "$ref": "#/definitions/string_types" - }, - { - "$ref": "#/definitions/int_types" - }, - { - "$ref": "#/definitions/number_types" - }, - { - "$ref": "#/definitions/date_types" - }, - { - "$ref": "#/definitions/boolean_types" - }, - { - "$ref": "#/definitions/struct_types" - } - ] - } - } - }, - "additionalProperties": false -} diff --git a/terraform/environments/data-platform/environment-configurations.tf b/terraform/environments/data-platform/environment-configurations.tf deleted file mode 100644 index 9a34ae9d1ea..00000000000 --- a/terraform/environments/data-platform/environment-configurations.tf +++ /dev/null @@ -1,21 +0,0 @@ -locals { - environment_configuration = local.environment_configurations[local.environment] - environment_configurations = { - development = { - apps_tools_account_id = local.environment_management.account_ids["data-platform-apps-and-tools-development"] - observability_platform_account_id = local.environment_management.account_ids["observability-platform-development"] - } - test = { - apps_tools_account_id = local.environment_management.account_ids["data-platform-apps-and-tools-development"] - observability_platform_account_id = local.environment_management.account_ids["observability-platform-development"] - } - preproduction = { - apps_tools_account_id = local.environment_management.account_ids["data-platform-apps-and-tools-development"] - observability_platform_account_id = local.environment_management.account_ids["observability-platform-development"] - } - production = { - apps_tools_account_id = local.environment_management.account_ids["data-platform-apps-and-tools-production"] - observability_platform_account_id = local.environment_management.account_ids["observability-platform-production"] - } - } -} diff --git a/terraform/environments/data-platform/glue.tf b/terraform/environments/data-platform/glue.tf deleted file mode 100644 index d093fe36e1f..00000000000 --- a/terraform/environments/data-platform/glue.tf +++ /dev/null @@ -1,190 +0,0 @@ -# creates database and tables in the glue catalogue for data as a product logs. -# Meaning logs are queryable via Athena. -resource "aws_glue_catalog_database" "data_product_logs" { - name = "daap_logs" -} - -resource "aws_glue_catalog_table" "lambdas" { - name = "lambdas" - database_name = aws_glue_catalog_database.data_product_logs.name - table_type = "EXTERNAL_TABLE" - - parameters = { - EXTERNAL = "TRUE" - comment = "table for logs from the python daap lambda functions" - - } - - storage_descriptor { - location = "s3://${module.logs_s3_bucket.bucket.id}/logs/json/" - input_format = "org.apache.hadoop.mapred.TextInputFormat" - output_format = "org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat" - - ser_de_info { - parameters = { - "serialization.format" = "1" - } - serialization_library = "org.openx.data.jsonserde.JsonSerDe" - } - - columns { - name = "lambda_name" - type = "string" - } - columns { - name = "data_product_name" - type = "string" - } - columns { - name = "table_name" - type = "string" - } - columns { - name = "date_time" - type = "string" - } - columns { - name = "function_name" - type = "string" - } - columns { - name = "image_version" - type = "string" - } - columns { - name = "base_image_version" - type = "string" - } - columns { - name = "level" - type = "string" - } - columns { - name = "message" - type = "string" - } - } - -} - - -resource "aws_glue_catalog_table" "s3_objects" { - name = "s3_objects" - database_name = aws_glue_catalog_database.data_product_logs.name - - table_type = "EXTERNAL_TABLE" - - parameters = { - EXTERNAL = "TRUE" - classification = "cloudtrail" - comment = "CloudTrail table for logs from the data and landing data buckets" - - } - - storage_descriptor { - location = "s3://${module.logs_s3_bucket.bucket.id}/AWSLogs/${data.aws_caller_identity.current.account_id}/CloudTrail/" - input_format = "com.amazon.emr.cloudtrail.CloudTrailInputFormat" - output_format = "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat" - - ser_de_info { - parameters = { - "serialization.format" = "1" - } - serialization_library = "org.apache.hive.hcatalog.data.JsonSerDe" - } - - columns { - name = "eventversion" - type = "string" - } - columns { - name = "useridentity" - type = "struct,sessionissuer:struct,ec2roledelivery:string,webidfederationdata:map>>" - } - columns { - name = "eventtime" - type = "string" - } - columns { - name = "eventsource" - type = "string" - } - columns { - name = "eventname" - type = "string" - } - columns { - name = "awsregion" - type = "string" - } - columns { - name = "sourceipaddress" - type = "string" - } - columns { - name = "useragent" - type = "string" - } - columns { - name = "errorcode" - type = "string" - } - columns { - name = "errormessage" - type = "string" - } - columns { - name = "requestparameters" - type = "string" - } - columns { - name = "responseelements" - type = "string" - } - columns { - name = "additionaleventdata" - type = "string" - } - columns { - name = "requestid" - type = "string" - } - columns { - name = "eventid" - type = "string" - } - columns { - name = "resources" - type = "array>" - } - columns { - name = "eventtype" - type = "string" - } - columns { - name = "apiversion" - type = "string" - } - columns { - name = "readonly" - type = "string" - } - columns { - name = "recipientaccountid" - type = "string" - } - columns { - name = "serviceeventdetails" - type = "string" - } - columns { - name = "sharedeventid" - type = "string" - } - columns { - name = "vpcendpointid" - type = "string" - } - - } -} diff --git a/terraform/environments/data-platform/iam-policies.tf b/terraform/environments/data-platform/iam-policies.tf deleted file mode 100644 index 0aed55aa600..00000000000 --- a/terraform/environments/data-platform/iam-policies.tf +++ /dev/null @@ -1,25 +0,0 @@ -// TODO Scope this down... - -data "aws_iam_policy_document" "openmetadata" { - statement { - sid = "openmetadata" - effect = "Allow" - actions = [ - "s3:*", - "athena:*", - "glue:*" - ] - resources = ["*"] - } -} - -module "openmetadata_iam_policy" { - source = "terraform-aws-modules/iam/aws//modules/iam-policy" - version = "~> 5.0" - - name_prefix = "openmetadata" - - policy = data.aws_iam_policy_document.openmetadata.json - - tags = local.tags -} \ No newline at end of file diff --git a/terraform/environments/data-platform/iam-roles.tf b/terraform/environments/data-platform/iam-roles.tf deleted file mode 100644 index 868455e84f9..00000000000 --- a/terraform/environments/data-platform/iam-roles.tf +++ /dev/null @@ -1,18 +0,0 @@ -module "openmetadata_iam_role" { - source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role" - version = "~> 5.0" - - create_role = true - - role_name = "openmetadata" - role_requires_mfa = false - - trusted_role_arns = ["arn:aws:iam::${local.environment_configuration.apps_tools_account_id}:root"] - - custom_role_policy_arns = [ - "arn:aws:iam::aws:policy/service-role/AWSQuicksightAthenaAccess", - module.openmetadata_iam_policy.arn - ] - - tags = local.tags -} diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf deleted file mode 100644 index a489a64b9c7..00000000000 --- a/terraform/environments/data-platform/iam.tf +++ /dev/null @@ -1,731 +0,0 @@ -# IAM policy documents for lambda functions - -data "aws_iam_policy_document" "log_to_bucket" { - statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket" - ] - resources = [ - "${module.logs_s3_bucket.bucket.arn}", - "${module.logs_s3_bucket.bucket.arn}/*" - ] - } -} - -data "aws_iam_policy_document" "manage_glue_databases" { - statement { - sid = "gluePermissions" - effect = "Allow" - actions = [ - "glue:BatchCreatePartition", - "glue:BatchDeletePartition", - "glue:BatchDeleteTable", - "glue:BatchDeleteTableVersion", - "glue:BatchGetPartition", - "glue:CreateDatabase", - "glue:CreatePartition", - "glue:CreatePartitionIndex", - "glue:CreateTable", - "glue:DeletePartition", - "glue:DeletePartitionIndex", - "glue:DeleteSchema", - "glue:DeleteTable", - "glue:GetDatabase", - "glue:GetDatabases", - "glue:GetPartition", - "glue:GetPartitionIndexes", - "glue:GetPartitions", - "glue:GetSchema", - "glue:GetSchemaByDefinition", - "glue:GetSchemaVersion", - "glue:GetSchemaVersionsDiff", - "glue:GetTable", - "glue:GetTables", - "glue:GetTableVersion", - "glue:GetTableVersions", - "glue:ListSchemas", - "glue:UpdatePartition", - "glue:UpdateRegistry", - "glue:UpdateSchema", - "glue:UpdateTable" - ] - resources = [ - "*" - ] - } -} - -data "aws_iam_policy_document" "read_openmetadata_secrets" { - statement { - sid = "openmetdataSecretsManager" - effect = "Allow" - actions = ["secretsmanager:GetSecretValue"] - resources = [aws_secretsmanager_secret.openmetadata.id] - } -} -data "aws_iam_policy_document" "read_metadata" { - statement { - sid = "s3ReadMetadata" - effect = "Allow" - actions = ["s3:GetObject", "s3:ListBucket"] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}/*", - "${module.metadata_s3_bucket.bucket.arn}", - ] - } -} - -data "aws_iam_policy_document" "write_metadata" { - statement { - sid = "s3WriteMetadata" - effect = "Allow" - actions = ["s3:PutObject"] - resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] - } - statement { - sid = "InvokePushToCatalogueLambda" - effect = "Allow" - actions = ["lambda:InvokeFunction"] - resources = [module.data_product_push_to_catalogue_lambda.lambda_function_arn] - } -} - -data "aws_iam_policy_document" "create_write_lambda_logs" { - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } -} - -data "aws_iam_policy_document" "athena_query_access" { - statement { - sid = "AthenaQueryAccess" - actions = [ - "athena:StartQueryExecution", - "athena:GetQueryExecution", - "athena:GetQueryResults", - ] - resources = [ - aws_athena_workgroup.data_product_athena_workgroup.arn - ] - } - - statement { - sid = "s3GetListPutAthenaQueryResults" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket", - "s3:PutObject", - "s3:GetBucketLocation" - ] - resources = [ - "${module.s3_athena_query_results_bucket.bucket.arn}", - "${module.s3_athena_query_results_bucket.bucket.arn}/*", - ] - } -} - -data "aws_iam_policy_document" "athena_load_lambda_function_policy" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - data.aws_iam_policy_document.manage_glue_databases.json, - data.aws_iam_policy_document.athena_query_access.json, - ] - - statement { - sid = "s3Access" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket", - "s3:GetBucketLocation" - ] - resources = [ - "${module.data_s3_bucket.bucket.arn}/raw/*", - "${module.data_s3_bucket.bucket.arn}/fail/*", - "${module.data_s3_bucket.bucket.arn}/curated/*", - "${module.data_s3_bucket.bucket.arn}", - ] - } -} - -data "aws_iam_policy_document" "landing_to_raw_lambda_policy" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "getDeleteLandingData" - effect = "Allow" - actions = [ - "s3:GetObject*", - "s3:DeleteObject*", - "s3:ListBucket", - ] - resources = [ - "${module.data_landing_s3_bucket.bucket.arn}/*", - "${module.data_landing_s3_bucket.bucket.arn}", - ] - } - - statement { - sid = "copyToRawFail" - effect = "Allow" - actions = [ - "s3:PutObject", - "s3:PutObjectTagging", - "s3:ListBucket", - - ] - resources = [ - "${module.data_s3_bucket.bucket.arn}/raw/*", - "${module.data_s3_bucket.bucket.arn}/raw", - "${module.data_s3_bucket.bucket.arn}/fail/*", - "${module.data_s3_bucket.bucket.arn}/fail", - ] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_authorizer_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] -} - -data "aws_iam_policy_document" "iam_policy_document_for_presigned_url_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "GetPutDataObject" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket", - ] - resources = [ - "${module.data_landing_s3_bucket.bucket.arn}/landing/*", - "${module.logs_s3_bucket.bucket.arn}/logs/*", - "${module.data_landing_s3_bucket.bucket.arn}/landing", - "${module.logs_s3_bucket.bucket.arn}/logs", - ] - } -} - -# API Gateway authoriser IAM permissions - -data "aws_iam_policy_document" "apigateway_trust_policy" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["apigateway.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -resource "aws_iam_role" "authoriser_role" { - name = "authoriser_role_${local.environment}" - assume_role_policy = data.aws_iam_policy_document.apigateway_trust_policy.json - tags = local.tags -} - -data "aws_iam_policy_document" "allow_invoke_authoriser_lambda_doc" { - statement { - effect = "Allow" - actions = ["lambda:InvokeFunction"] - resources = [module.data_product_authorizer_lambda.lambda_function_arn] - } -} - -resource "aws_iam_policy" "allow_invoke_authoriser_lambda" { - name = "allow_invoke_authoriser_lambda" - path = "/" - policy = data.aws_iam_policy_document.allow_invoke_authoriser_lambda_doc.json - tags = local.tags -} - -resource "aws_iam_role_policy_attachment" "attach_allow_invoke_authoriser_lambda" { - role = aws_iam_role.authoriser_role.name - policy_arn = aws_iam_policy.allow_invoke_authoriser_lambda.arn -} - -# S3 policy - -data "aws_iam_policy_document" "data_s3_bucket_policy_document" { - statement { - sid = "DenyNonFullControlObjects" - effect = "Deny" - actions = ["s3:PutObject"] - resources = ["${module.data_s3_bucket.bucket.arn}/*"] - - principals { - identifiers = ["*"] - type = "AWS" - } - - condition { - test = "StringNotEquals" - variable = "s3:x-amz-acl" - - values = [ - "bucket-owner-full-control" - ] - } - } - -} - -data "aws_iam_policy_document" "data_landing_s3_bucket_policy_document" { - statement { - sid = "DenyNonFullControlObjects" - effect = "Deny" - actions = ["s3:PutObject"] - resources = ["${module.data_landing_s3_bucket.bucket.arn}/*"] - - principals { - identifiers = ["*"] - type = "AWS" - } - - condition { - test = "StringNotEquals" - variable = "s3:x-amz-acl" - - values = [ - "bucket-owner-full-control" - ] - } - } - -} - -data "aws_iam_policy_document" "metadata_s3_bucket_policy_document" { - statement { - sid = "DenyNonFullControlObjects" - effect = "Deny" - actions = ["s3:PutObject"] - resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] - - principals { - identifiers = ["*"] - type = "AWS" - } - - condition { - test = "StringNotEquals" - variable = "s3:x-amz-acl" - - values = [ - "bucket-owner-full-control" - ] - } - } - -} - -data "aws_iam_policy_document" "logs_s3_bucket_policy_document" { - statement { - sid = "AllowPutFromCloudtrail" - effect = "Allow" - - principals { - type = "Service" - identifiers = ["cloudtrail.amazonaws.com"] - } - - actions = [ - "s3:PutObject" - ] - - resources = [ - "${module.logs_s3_bucket.bucket.arn}/AWSLogs/${data.aws_caller_identity.current.account_id}/*" - ] - - condition { - test = "StringEquals" - variable = "s3:x-amz-acl" - - values = [ - "bucket-owner-full-control" - ] - } - condition { - test = "StringEquals" - variable = "aws:SourceArn" - - # If the arn is directly referenced there were issues with circular dependencies - # So the arn is predicted from it's component parts - values = ["arn:aws:cloudtrail:${local.region}:${local.account_id}:trail/data_platform_s3_putobject_trail_${local.environment}"] - } - } - - statement { - sid = "AWSCloudTrailAclCheck" - effect = "Allow" - - principals { - type = "Service" - identifiers = ["cloudtrail.amazonaws.com"] - } - - actions = [ - "s3:GetBucketAcl" - ] - - resources = [module.logs_s3_bucket.bucket.arn] - - condition { - test = "StringEquals" - variable = "aws:SourceArn" - - # If the arn is directly referenced there were issues with circular dependencies - # So the arn is predicted from it's component parts - values = [aws_cloudtrail.data_s3_put_objects.arn] - } - } - - statement { - sid = "DenyNonFullControlObjects" - effect = "Deny" - actions = ["s3:PutObject"] - resources = ["${module.logs_s3_bucket.bucket.arn}/*"] - - principals { - identifiers = ["*"] - type = "AWS" - } - - condition { - test = "StringNotEquals" - variable = "s3:x-amz-acl" - - values = [ - "bucket-owner-full-control" - ] - } - } - -} - -# api gateway create data product metdata permissions -data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.write_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [ - module.metadata_s3_bucket.bucket.arn - ] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [module.data_s3_bucket.bucket.arn] - } - statement { - sid = "InvokeAthenaLoadLambda" - effect = "Allow" - actions = ["lambda:InvokeFunction"] - resources = [module.data_product_athena_load_lambda.lambda_function_arn] - } - statement { - sid = "GlueGetTableDeleteTable" - effect = "Allow" - actions = [ - "glue:DeleteTable", - "glue:GetTables" - ] - resources = [ - "*" - ] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_get_schema_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] -} - -data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "ListBucket" - effect = "Allow" - actions = ["s3:ListBucket"] - resources = [ - module.data_s3_bucket.bucket.arn - ] - } - statement { - sid = "InvokeAthenaLoadLambda" - effect = "Allow" - actions = ["lambda:InvokeFunction"] - resources = [module.data_product_athena_load_lambda.lambda_function_arn] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_write_metadata_and_schema" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.write_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] -} - -data "aws_iam_policy_document" "iam_policy_document_for_update_schema" { - source_policy_documents = [ - data.aws_iam_policy_document.write_metadata.json, - data.aws_iam_policy_document.athena_load_lambda_function_policy.json - ] -} - -resource "aws_iam_role" "api_gateway_cloud_watch_role" { - name = "data_platform_apigateway_log_${local.environment}" - assume_role_policy = data.aws_iam_policy_document.apigateway_trust_policy.json - tags = local.tags -} - -data "aws_iam_policy_document" "cloudtrail_assume_role_policy" { - statement { - effect = "Allow" - - principals { - type = "Service" - identifiers = ["cloudtrail.amazonaws.com"] - } - - actions = ["sts:AssumeRole"] - } -} - -data "aws_iam_policy_document" "cloudtrail_cloudwatch_policy" { - statement { - sid = "cloudtrailToCloudwatch" - effect = "Allow" - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = [ - "arn:aws:logs:${local.region}:${local.account_id}:log-group:${aws_cloudwatch_log_group.data_platform_s3_putobject_trail.name}:log-stream:${local.account_id}_CloudTrail_${local.region}*", - ] - } -} - -resource "aws_iam_policy" "cloudtrail_cloudwatch_policy" { - name = "data_platform_cloudtrail_cloudwatch_policy_${local.environment}" - policy = data.aws_iam_policy_document.cloudtrail_cloudwatch_policy.json -} - -resource "aws_iam_role" "cloudtrail_cloudwatch_role" { - name = "data_platform_cloudtrail_log_${local.environment}" - assume_role_policy = data.aws_iam_policy_document.cloudtrail_assume_role_policy.json - tags = local.tags -} - -resource "aws_iam_role_policy_attachment" "cloudtrail_cloudwatch" { - role = aws_iam_role.cloudtrail_cloudwatch_role.id - policy_arn = aws_iam_policy.cloudtrail_cloudwatch_policy.arn -} - -resource "aws_iam_role_policy_attachment" "api_gateway_cloudwatchlogs" { - role = aws_iam_role.api_gateway_cloud_watch_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs" -} - -data "aws_iam_policy_document" "iam_policy_document_for_preview_data" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - data.aws_iam_policy_document.athena_query_access.json, - ] - statement { - sid = "s3Access" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket" - ] - resources = [ - "${module.data_s3_bucket.bucket.arn}/curated/*", - "${module.data_s3_bucket.bucket.arn}" - ] - } - statement { - sid = "GluePermissions" - effect = "Allow" - actions = [ - "glue:GetTable", - "glue:GetPartitions", - "glue:GetPartition", - "glue:GetDatabases", - "glue:GetDatabase" - ] - resources = [ - "*" - ] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_delete_table_for_data_product_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.write_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - data.aws_iam_policy_document.manage_glue_databases.json, - data.aws_iam_policy_document.athena_query_access.json, - ] - - statement { - sid = "s3ListGetDeletePutRawFailCurated" - effect = "Allow" - actions = [ - "s3:PutObject", - "s3:ListBucket", - "s3:GetObject", - "s3:DeleteObject" - ] - resources = [ - "${module.data_s3_bucket.bucket.arn}", - "${module.data_s3_bucket.bucket.arn}/raw/*", - "${module.data_s3_bucket.bucket.arn}/fail/*", - "${module.data_s3_bucket.bucket.arn}/curated/*", - "${module.data_s3_bucket.bucket.arn}/raw", - "${module.data_s3_bucket.bucket.arn}/fail", - "${module.data_s3_bucket.bucket.arn}/curated", - ] - } - - statement { - sid = "s3ListDeleteSchema" - effect = "Allow" - actions = [ - "s3:DeleteObject" - ] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}/*", - "${module.metadata_s3_bucket.bucket.arn}" - ] - } - - statement { - sid = "GlueGetTableDeleteTable" - effect = "Allow" - actions = [ - "glue:DeleteTable", - "glue:GetTable" - ] - resources = [ - "*" - ] - } -} - -data "aws_iam_policy_document" "iam_policy_document_for_push_to_catalogue_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_openmetadata_secrets.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] -} - -data "aws_iam_policy_document" "iam_policy_document_for_delete_data_product_lambda" { - source_policy_documents = [ - data.aws_iam_policy_document.log_to_bucket.json, - data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.write_metadata.json, - data.aws_iam_policy_document.create_write_lambda_logs.json, - ] - - statement { - sid = "s3ListDeleteRawFailCurated" - effect = "Allow" - actions = [ - "s3:ListBucket", - "s3:DeleteObject" - ] - resources = [ - "${module.data_s3_bucket.bucket.arn}", - "${module.data_s3_bucket.bucket.arn}/raw/*", - "${module.data_s3_bucket.bucket.arn}/fail/*", - "${module.data_s3_bucket.bucket.arn}/curated/*", - "${module.data_s3_bucket.bucket.arn}/raw", - "${module.data_s3_bucket.bucket.arn}/fail", - "${module.data_s3_bucket.bucket.arn}/curated", - ] - } - - statement { - sid = "s3ListDeleteMetadata" - effect = "Allow" - actions = [ - "s3:ListBucket", - "s3:DeleteObject" - ] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}/*", - "${module.metadata_s3_bucket.bucket.arn}" - ] - } - - statement { - sid = "GlueGetDeleteDatabase" - effect = "Allow" - actions = [ - "glue:GetDatabase", - "glue:DeleteDatabase", - ] - resources = [ - "*" - ] - } -} diff --git a/terraform/environments/data-platform/infra_route53.tf b/terraform/environments/data-platform/infra_route53.tf deleted file mode 100644 index 41d18b92072..00000000000 --- a/terraform/environments/data-platform/infra_route53.tf +++ /dev/null @@ -1,277 +0,0 @@ -################################################## -# Production -################################################## - -resource "aws_route53_zone" "data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - name = "data-platform.service.justice.gov.uk" - tags = local.tags -} - -# delegating to data-platform-apps-and-tools-production -resource "aws_route53_record" "delegate_apps_tools_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "apps-tools.data-platform.service.justice.gov.uk" - type = "NS" - ttl = "300" - records = [ - "ns-1707.awsdns-21.co.uk.", - "ns-492.awsdns-61.com.", - "ns-1189.awsdns-20.org.", - "ns-581.awsdns-08.net." - ] -} - -# Delegating to data-platform-development -resource "aws_route53_record" "delegate_development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "development.data-platform.service.justice.gov.uk" - type = "NS" - ttl = "300" - records = [ - "ns-1741.awsdns-25.co.uk.", - "ns-446.awsdns-55.com.", - "ns-1406.awsdns-47.org.", - "ns-952.awsdns-55.net." - ] -} - -# Delegating to data-platform-preproduction -resource "aws_route53_record" "delegate_preproduction_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "preproduction.data-platform.service.justice.gov.uk" - type = "NS" - ttl = "300" - records = [ - "ns-328.awsdns-41.com.", - "ns-1671.awsdns-16.co.uk.", - "ns-792.awsdns-35.net.", - "ns-1106.awsdns-10.org." - ] -} - -# Delegating to data-platform-test -resource "aws_route53_record" "delegate_test_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "test.data-platform.service.justice.gov.uk" - type = "NS" - ttl = "300" - records = [ - "ns-407.awsdns-50.com.", - "ns-1837.awsdns-37.co.uk.", - "ns-681.awsdns-21.net.", - "ns-1302.awsdns-34.org." - ] -} - -# Technical Documentation on GitHub Pages -resource "aws_route53_record" "data_platform_technical_documentation" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "technical-documentation.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ministryofjustice.github.io."] -} - -# User Guidance on GitHub Pages -resource "aws_route53_record" "data_platform_user_guidance" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "user-guidance.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ministryofjustice.github.io."] -} - -# Front Door on GitHub Pages -resource "aws_route53_record" "data_platform_front_door" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "data-platform.service.justice.gov.uk" - type = "A" - ttl = "300" - records = ["185.199.108.153", "185.199.109.153", "185.199.110.153", "185.199.111.153"] -} - -# PagerDuty Status Page (HTTP Traffic) -resource "aws_route53_record" "http_traffic_status_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "status.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["cd-4a9d4d61b9ba517b444f76f11a406278.hosted-status.pagerduty.com"] -} - -# PagerDuty Status Page (TLS Validation) -resource "aws_route53_record" "tls_validation_status_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "_701f6977b827d5ad23c4f98802a51bc3.status.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["_56473aa9b1f7b9aec52ac3d3ea416721.yygwskclfy.acm-validations.aws."] -} - -# PagerDuty Status Page (DKIM 1) -resource "aws_route53_record" "dkim_one_status_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "pdt._domainkey.status.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["pdt.domainkey.u31181182.wl183.sendgrid.net."] -} - -# PagerDuty Status Page (DKIM 2) -resource "aws_route53_record" "dkim_two_status_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "pdt2._domainkey.status.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["pdt2.domainkey.u31181182.wl183.sendgrid.net."] -} - -# PagerDuty Status Page (Mail CNAME) -resource "aws_route53_record" "mail_cname_status_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "em9648.status.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["u31181182.wl183.sendgrid.net."] -} - -# Delegating to data-platform-apps-and-tools-production -resource "aws_route53_record" "delegate_assets_production_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "assets.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ingress.apps-tools.data-platform.service.justice.gov.uk."] -} - -# Delegating to data-platform-apps-and-tools-production -resource "aws_route53_record" "delegate_control_panel_production_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "control-panel.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ingress.apps-tools.data-platform.service.justice.gov.uk."] -} - -# Auth0 -resource "aws_route53_record" "auth_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-production" ? 1 : 0 - - zone_id = aws_route53_zone.data_platform_service_justice_gov_uk[0].zone_id - name = "auth.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ministryofjustice-data-platform-cd-dk5jlhurgfon6kxk.edge.tenants.uk.auth0.com."] -} - -################################################## -# Development -################################################## - -resource "aws_route53_zone" "development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-development" ? 1 : 0 - - name = "development.data-platform.service.justice.gov.uk" - tags = local.tags -} - -# Delegating to data-platform-apps-and-tools-development -resource "aws_route53_record" "delegate_apps_tools_development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-development" ? 1 : 0 - - zone_id = aws_route53_zone.development_data_platform_service_justice_gov_uk[0].zone_id - name = "apps-tools.development.data-platform.service.justice.gov.uk" - type = "NS" - ttl = "300" - records = [ - "ns-1673.awsdns-17.co.uk", - "ns-1230.awsdns-25.org", - "ns-122.awsdns-15.com", - "ns-876.awsdns-45.net" - ] -} - -# Delegating to data-platform-apps-and-tools-development -resource "aws_route53_record" "delegate_assets_development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-development" ? 1 : 0 - - zone_id = aws_route53_zone.development_data_platform_service_justice_gov_uk[0].zone_id - name = "assets.development.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ingress.apps-tools.development.data-platform.service.justice.gov.uk."] -} - -# Delegating to data-platform-apps-and-tools-development -resource "aws_route53_record" "delegate_control_panel_development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-development" ? 1 : 0 - - zone_id = aws_route53_zone.development_data_platform_service_justice_gov_uk[0].zone_id - name = "control-panel.development.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ingress.apps-tools.development.data-platform.service.justice.gov.uk."] -} - -# Auth0 -resource "aws_route53_record" "auth_development_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-development" ? 1 : 0 - - zone_id = aws_route53_zone.development_data_platform_service_justice_gov_uk[0].zone_id - name = "auth.development.data-platform.service.justice.gov.uk" - type = "CNAME" - ttl = "300" - records = ["ministryofjustice-data-platform-development-cd-zvdb1vq1ynemnuqs.edge.tenants.uk.auth0.com."] -} - -################################################## -# PreProduction -################################################## - -resource "aws_route53_zone" "preproduction_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-preproduction" ? 1 : 0 - - name = "preproduction.data-platform.service.justice.gov.uk" - tags = local.tags -} - -################################################## -# Test -################################################## - -resource "aws_route53_zone" "test_data_platform_service_justice_gov_uk" { - count = terraform.workspace == "data-platform-test" ? 1 : 0 - - name = "test.data-platform.service.justice.gov.uk" - tags = local.tags -} diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf deleted file mode 100644 index 990358c00c7..00000000000 --- a/terraform/environments/data-platform/lambda.tf +++ /dev/null @@ -1,492 +0,0 @@ -module "data_product_docs_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_docs" - tags = local.tags - description = "Lambda for swagger api docs" - function_name = "data_product_docs_${local.environment}" - role_name = "docs_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.create_write_lambda_logs.json - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-docs-lambda-ecr-repo:${local.docs_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" - } - } - -} - -module "data_product_authorizer_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_authorizer" - tags = local.tags - description = "Lambda for custom API Gateway authorizer" - role_name = "authorizer_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_authorizer_lambda.json - function_name = "data_product_authorizer_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-authorizer-lambda-ecr-repo:${local.authorizer_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, { - authorizationToken = local.api_auth_token - api_resource_arn = "${aws_api_gateway_rest_api.data_platform.execution_arn}/*/*" - }) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" - } - } - -} - -module "data_product_landing_to_raw_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_landing_to_raw" - tags = local.tags - description = "Lambda to retrieve Glue metadata for a specified table in a database" - role_name = "landing_to_raw_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.landing_to_raw_lambda_policy.json - function_name = "data_product_landing_to_raw_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-landing-to-raw-lambda-ecr-repo:${local.landing_to_raw_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) - - allowed_triggers = { - - AllowExecutionFromCloudWatch = { - action = "lambda:InvokeFunction" - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.object_created_data_landing.arn - } - } - -} - -module "data_product_presigned_url_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_presigned_url" - tags = local.tags - description = "Lambda to generate a presigned url for uploading data" - role_name = "presigned_url_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_presigned_url_lambda.json - function_name = "data_product_presigned_url_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-presigned-url-lambda-ecr-repo:${local.presigned_url_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.upload_data_for_data_product_table_name.http_method}${aws_api_gateway_resource.upload_data_for_data_product_table_name.path}" - } - } - -} - -module "data_product_athena_load_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_athena_load" - tags = local.tags - description = "Lambda to load and transform raw data products landing in s3. Creates partitioned parquet tables" - role_name = "athena_load_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.athena_load_lambda_function_policy.json - function_name = "data_product_athena_load_${local.environment}" - create_role = true - reserved_concurrent_executions = 100 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-athena-load-lambda-ecr-repo:${local.athena_load_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { - ENVIRONMENT = local.environment - }) - - allowed_triggers = { - - AllowExecutionFromCloudWatch = { - action = "lambda:InvokeFunction" - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.object_created_raw_data.arn - } - } - -} - - -module "data_product_create_metadata_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_create_metadata" - tags = local.tags - description = "Lambda to create the first version of a json metadata file for a data product" - role_name = "data_product_metadata_lambda_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_create_metadata_lambda.json - function_name = "data_product_create_metadata_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-create-metadata-lambda-ecr-repo:${local.create_metadata_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { - ENVIRONMENT = local.environment, PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn - }) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.register_data_product.http_method}${aws_api_gateway_resource.register_data_product.path}" - } - } - -} - -module "reload_data_product_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "reload_data_product" - tags = local.tags - description = "Reload the data in a data product from raw history to curated, and recreate the athena tables." - role_name = "reload_data_product_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_reload_data_product_lambda.json - function_name = "reload_data_product_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-reload-data-product-lambda-ecr-repo:${local.reload_data_product_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { - ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name - }) - -} - -module "resync_unprocessed_files_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "resync_unprocessed_files" - tags = local.tags - description = "Retrigger the athena load for extraction timestamps in raw history and not in curated data, for one data product" - role_name = "resync_unprocessed_files_role_${local.environment}" - policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_resync_unprocessed_files_lambda.json - function_name = "resync_unprocessed_files_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-resync-unprocessed-files-lambda-ecr-repo:${local.resync_unprocessed_files_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 512 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, { - ATHENA_LOAD_LAMBDA = module.data_product_athena_load_lambda.lambda_function_name - }) - -} - -module "data_product_create_schema_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_create_schema" - tags = local.tags - description = "Lambda to create the first version of a json schema file for a data product" - role_name = "data_product_schema_lambda_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_write_metadata_and_schema.json - policy_json_attached = true - function_name = "data_product_create_schema_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-create-schema-lambda-ecr-repo:${local.create_schema_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, - { PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn } - ) - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" - } - } -} - -module "get_schema_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "get_schema" - tags = local.tags - description = "Fetch the schema for a table from S3" - role_name = "get_schema_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_get_schema_lambda.json - policy_json_attached = true - function_name = "get_schema_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-get-schema-lambda-ecr-repo:${local.get_schema_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.get_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" - } - } -} - -module "data_product_update_metadata_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_update_metadata" - tags = local.tags - description = "Fetch the schema for a table from S3" - role_name = "update_metadata_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_write_metadata_and_schema.json - policy_json_attached = true - function_name = "data_product_update_metadata_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-update-metadata-lambda-ecr-repo:${local.update_metadata_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, - { PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn } - ) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.update_data_product.http_method}${aws_api_gateway_resource.data_product_name.path}" - } - } -} - -module "data_product_update_schema_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_update_schema" - tags = local.tags - description = "Update the schema for a table in a data product" - role_name = "update_schema_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_update_schema.json - policy_json_attached = true - function_name = "data_product_update_schema_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-update-schema-lambda-ecr-repo:${local.update_schema_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, - { PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn } - ) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.update_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" - } - } -} - -module "preview_data_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_preview_data" - tags = local.tags - description = "Preview small sample of data through athena " - role_name = "preview_data_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_preview_data.json - policy_json_attached = true - function_name = "data_product_preview_data_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-preview-data-lambda-ecr-repo:${local.preview_data_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.preview_data_from_data_product.http_method}${aws_api_gateway_resource.data_product_preview.path}" - } - } -} - -module "delete_table_for_data_product_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "delete_table" - tags = local.tags - description = "Delete table and data for a data product" - role_name = "delete_table_for_data_product_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_delete_table_for_data_product_lambda.json - policy_json_attached = true - function_name = "delete_table_for_data_product_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-delete-table-lambda-ecr-repo:${local.delete_table_for_data_product_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, - { PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn } - ) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.delete_table_for_data_product.http_method}${aws_api_gateway_resource.data_product_table_name.path}" - } - } -} - -module "data_product_push_to_catalogue_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "data_product_push_to_catalogue" - tags = local.tags - description = "Pushes metadata to openmetadata catalogue" - role_name = "push_to_catalogue_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_push_to_catalogue_lambda.json - policy_json_attached = true - function_name = "data_product_push_to_catalogue_${local.environment}" - create_role = true - reserved_concurrent_executions = 100 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-push-to-catalogue-lambda-ecr-repo:${local.push_to_catalogue_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, local.openmetadata_environment_vars) - - allowed_triggers = { - - AllowExecutionFromCreateMetadataLambda = { - action = "lambda:InvokeFunction" - principal = "lambda.amazonaws.com" - source_arn = module.data_product_create_metadata_lambda.lambda_function_arn - } - AllowExecutionFromCreateSchemaLambda = { - action = "lambda:InvokeFunction" - principal = "lambda.amazonaws.com" - source_arn = module.data_product_create_schema_lambda.lambda_function_arn - } - AllowExecutionFromUpdateSchemaLambda = { - action = "lambda:InvokeFunction" - principal = "lambda.amazonaws.com" - source_arn = module.data_product_update_schema_lambda.lambda_function_arn - } - AllowExecutionFromUpdateMetadataLambda = { - action = "lambda:InvokeFunction" - principal = "lambda.amazonaws.com" - source_arn = module.data_product_update_metadata_lambda.lambda_function_arn - } - AllowExecutionFromDeleteTableLambda = { - action = "lambda:InvokeFunction" - principal = "lambda.amazonaws.com" - source_arn = module.delete_table_for_data_product_lambda.lambda_function_arn - } - } -} - -module "delete_data_product_lambda" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 - application_name = "delete_data_product" - tags = local.tags - description = "Delete a data product" - role_name = "delete_data_product_role_${local.environment}" - policy_json = data.aws_iam_policy_document.iam_policy_document_for_delete_data_product_lambda.json - policy_json_attached = true - function_name = "delete_data_product_${local.environment}" - create_role = true - reserved_concurrent_executions = 1 - - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-delete-data-product-lambda-ecr-repo:${local.delete_data_product_version}" - timeout = 600 - tracing_mode = "Active" - memory_size = 128 - - environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars, - { PUSH_TO_CATALOGUE_LAMBDA_ARN = module.data_product_push_to_catalogue_lambda.lambda_function_arn } - ) - - allowed_triggers = { - - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.delete_data_product.http_method}${aws_api_gateway_resource.data_product_name.path}" - } - } -} diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf deleted file mode 100644 index b1fec6f0cf6..00000000000 --- a/terraform/environments/data-platform/locals.tf +++ /dev/null @@ -1,62 +0,0 @@ -#### This file can be used to store locals specific to the member account #### -locals { - lambda_runtime = "python3.9" - lambda_timeout_in_seconds = 15 - region = "eu-west-2" - account_id = local.environment_management.account_ids[terraform.workspace] - api_auth_token = jsondecode(data.aws_secretsmanager_secret_version.api_auth.secret_string)["auth-token"] - - - # Glue - glue_default_arguments = { - "--job-bookmark-option" = "job-bookmark-disable" - "--enable-continuous-cloudwatch-log" = "true" - "--enable-continuous-log-filter" = "true" - "--enable-glue-datacatalog" = "true" - "--enable-job-insights" = "true" - "--enable-continuous-log-filter" = "true" - } - name = "data-platform-product" - glue_version = "4.0" - max_retries = 0 - worker_type = "G.1X" - number_of_workers = 2 - timeout = 120 # minutes - execution_class = "STANDARD" - max_concurrent = 5 - glue_log_group_retention_in_days = 7 - - docs_version = lookup(var.docs_versions, local.environment) - authorizer_version = lookup(var.authorizer_versions, local.environment) - presigned_url_version = lookup(var.presigned_url_versions, local.environment) - athena_load_version = lookup(var.athena_load_versions, local.environment) - create_metadata_version = lookup(var.create_metadata_versions, local.environment) - resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment) - reload_data_product_version = lookup(var.reload_data_product_versions, local.environment) - get_schema_version = lookup(var.get_schema_versions, local.environment) - create_schema_version = lookup(var.create_schema_versions, local.environment) - landing_to_raw_version = lookup(var.landing_to_raw_versions, local.environment) - update_metadata_version = lookup(var.update_metadata_versions, local.environment) - update_schema_version = lookup(var.update_schema_versions, local.environment) - preview_data_version = lookup(var.preview_data_versions, local.environment) - delete_table_for_data_product_version = lookup(var.delete_table_for_data_product_versions, local.environment) - push_to_catalogue_version = lookup(var.push_to_catalogue_versions, local.environment) - delete_data_product_version = lookup(var.delete_data_product_versions, local.environment) - - # Environment vars that are used by many lambdas - logger_environment_vars = { - LOG_BUCKET = module.logs_s3_bucket.bucket.id - } - - storage_environment_vars = { - RAW_DATA_BUCKET = module.data_s3_bucket.bucket.id - CURATED_DATA_BUCKET = module.data_s3_bucket.bucket.id - METADATA_BUCKET = module.metadata_s3_bucket.bucket.id - LANDING_ZONE_BUCKET = module.data_landing_s3_bucket.bucket.id - } - - openmetadata_environment_vars = { - OPENMETADATA_JWT_SECRET_ARN = aws_secretsmanager_secret.openmetadata.id - OPENMETADATA_DEV_API_URL = "https://catalogue.apps-tools.development.data-platform.service.justice.gov.uk/api" - } -} diff --git a/terraform/environments/data-platform/networking.auto.tfvars.json b/terraform/environments/data-platform/networking.auto.tfvars.json deleted file mode 100644 index eb562e997b6..00000000000 --- a/terraform/environments/data-platform/networking.auto.tfvars.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "networking": [ - { - "business-unit": "platforms", - "set": "general", - "application": "data-platform" - } - ] -} diff --git a/terraform/environments/data-platform/observability-platform.tf b/terraform/environments/data-platform/observability-platform.tf deleted file mode 100644 index de0898d0e92..00000000000 --- a/terraform/environments/data-platform/observability-platform.tf +++ /dev/null @@ -1,9 +0,0 @@ -module "observability_platform_tenant" { - #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions - - source = "ministryofjustice/observability-platform-tenant/aws" - version = "1.0.0" - - observability_platform_account_id = local.environment_configuration.observability_platform_account_id - enable_xray = true -} diff --git a/terraform/environments/data-platform/output.tf b/terraform/environments/data-platform/output.tf deleted file mode 100644 index 4a2d0b78468..00000000000 --- a/terraform/environments/data-platform/output.tf +++ /dev/null @@ -1,8 +0,0 @@ - -output "docs_endpoint" { - value = join("", [aws_api_gateway_deployment.deployment.invoke_url, aws_api_gateway_stage.default_stage.stage_name, "/docs/"]) -} - -output "presigned_url_endpoint" { - value = join("", [aws_api_gateway_deployment.deployment.invoke_url, aws_api_gateway_stage.default_stage.stage_name, "/presigned_url/"]) -} diff --git a/terraform/environments/data-platform/platform_backend.tf b/terraform/environments/data-platform/platform_backend.tf deleted file mode 100644 index 6f0d19ccf1c..00000000000 --- a/terraform/environments/data-platform/platform_backend.tf +++ /dev/null @@ -1,14 +0,0 @@ -# Backend -terraform { - # `backend` blocks do not support variables, so the following are hard-coded here: - # - S3 bucket name, which is created in modernisation-platform-account/s3.tf - backend "s3" { - acl = "bucket-owner-full-control" - bucket = "modernisation-platform-terraform-state" - dynamodb_table = "modernisation-platform-terraform-state-lock" - encrypt = true - key = "terraform.tfstate" - region = "eu-west-2" - workspace_key_prefix = "environments/members/data-platform" # This will store the object as environments/members/data-platform/${workspace}/terraform.tfstate - } -} diff --git a/terraform/environments/data-platform/platform_base_variables.tf b/terraform/environments/data-platform/platform_base_variables.tf deleted file mode 100644 index e5713ed9493..00000000000 --- a/terraform/environments/data-platform/platform_base_variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "networking" { - - type = list(any) - -} - -variable "collaborator_access" { - type = string - default = "developer" - description = "Collaborators must specify which access level they are using, eg set an environment variable of export TF_VAR_collaborator_access=migration" -} diff --git a/terraform/environments/data-platform/platform_data.tf b/terraform/environments/data-platform/platform_data.tf deleted file mode 100644 index 9844360a8cd..00000000000 --- a/terraform/environments/data-platform/platform_data.tf +++ /dev/null @@ -1,173 +0,0 @@ -# Current account data -data "aws_region" "current" {} - -data "aws_caller_identity" "current" {} - -# VPC and subnet data -data "aws_vpc" "shared" { - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}" - } -} - -data "aws_subnets" "shared-data" { - filter { - name = "vpc-id" - values = [data.aws_vpc.shared.id] - } - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-data*" - } -} - -data "aws_subnets" "shared-private" { - filter { - name = "vpc-id" - values = [data.aws_vpc.shared.id] - } - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-private*" - } -} - -data "aws_subnets" "shared-public" { - filter { - name = "vpc-id" - values = [data.aws_vpc.shared.id] - } - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-public*" - } -} - -data "aws_subnet" "data_subnets_a" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-data-${data.aws_region.current.name}a" - } -} - -data "aws_subnet" "data_subnets_b" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-data-${data.aws_region.current.name}b" - } -} - -data "aws_subnet" "data_subnets_c" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-data-${data.aws_region.current.name}c" - } -} - -data "aws_subnet" "private_subnets_a" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-private-${data.aws_region.current.name}a" - } -} - -data "aws_subnet" "private_subnets_b" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-private-${data.aws_region.current.name}b" - } -} - -data "aws_subnet" "private_subnets_c" { - vpc_id = data.aws_vpc.shared.id - tags = { - "Name" = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-private-${data.aws_region.current.name}c" - } -} - -data "aws_subnet" "public_subnets_a" { - vpc_id = data.aws_vpc.shared.id - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-public-${data.aws_region.current.name}a" - } -} - -data "aws_subnet" "public_subnets_b" { - vpc_id = data.aws_vpc.shared.id - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-public-${data.aws_region.current.name}b" - } -} - -data "aws_subnet" "public_subnets_c" { - vpc_id = data.aws_vpc.shared.id - tags = { - Name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}-public-${data.aws_region.current.name}c" - } -} - -# Route53 DNS data -data "aws_route53_zone" "external" { - provider = aws.core-vpc - - name = "${var.networking[0].business-unit}-${local.environment}.modernisation-platform.service.justice.gov.uk." - private_zone = false -} - -data "aws_route53_zone" "inner" { - provider = aws.core-vpc - - name = "${var.networking[0].business-unit}-${local.environment}.modernisation-platform.internal." - private_zone = true -} - -data "aws_route53_zone" "network-services" { - provider = aws.core-network-services - - name = "modernisation-platform.service.justice.gov.uk." - private_zone = false -} - -# Shared KMS keys (per business unit) -data "aws_kms_key" "general_shared" { - key_id = "arn:aws:kms:eu-west-2:${local.environment_management.account_ids["core-shared-services-production"]}:alias/general-${var.networking[0].business-unit}" -} - -data "aws_kms_key" "ebs_shared" { - key_id = "arn:aws:kms:eu-west-2:${local.environment_management.account_ids["core-shared-services-production"]}:alias/ebs-${var.networking[0].business-unit}" -} - -data "aws_kms_key" "rds_shared" { - key_id = "arn:aws:kms:eu-west-2:${local.environment_management.account_ids["core-shared-services-production"]}:alias/rds-${var.networking[0].business-unit}" -} - -# State for core-network-services resource information -data "terraform_remote_state" "core_network_services" { - backend = "s3" - config = { - acl = "bucket-owner-full-control" - bucket = "modernisation-platform-terraform-state" - key = "environments/accounts/core-network-services/core-network-services-production/terraform.tfstate" - region = "eu-west-2" - encrypt = "true" - } -} - -data "aws_organizations_organization" "root_account" {} - -# Retrieve information about the modernisation platform account -data "aws_caller_identity" "modernisation_platform" { - provider = aws.modernisation-platform -} - -# caller account information to instantiate aws.oidc provider -data "aws_caller_identity" "original_session" { - provider = aws.original-session -} - -data "aws_iam_session_context" "whoami" { - provider = aws.original-session - arn = data.aws_caller_identity.original_session.arn -} - -# Get the environments file from the main repository -data "http" "environments_file" { - url = "https://raw.githubusercontent.com/ministryofjustice/modernisation-platform/main/environments/${local.application_name}.json" -} diff --git a/terraform/environments/data-platform/platform_locals.tf b/terraform/environments/data-platform/platform_locals.tf deleted file mode 100644 index 4bca6941a3f..00000000000 --- a/terraform/environments/data-platform/platform_locals.tf +++ /dev/null @@ -1,38 +0,0 @@ -locals { - - application_name = "data-platform" - - environment_management = jsondecode(data.aws_secretsmanager_secret_version.environment_management.secret_string) - - # Stores modernisation platform account id for setting up the modernisation-platform provider - modernisation_platform_account_id = data.aws_ssm_parameter.modernisation_platform_account_id.value - - # This takes the name of the Terraform workspace (e.g. core-vpc-production), strips out the application name (e.g. core-vpc), and checks if - # the string leftover is `-production`, if it isn't (e.g. core-vpc-non-production => -non-production) then it sets the var to false. - is-production = substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-production" - is-preproduction = substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-preproduction" - is-test = substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-test" - is-development = substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-development" - - # Merge tags from the environment json file with additional ones - tags = merge( - jsondecode(data.http.environments_file.response_body).tags, - { "is-production" = local.is-production }, - { "environment-name" = terraform.workspace }, - { "source-code" = "https://github.com/ministryofjustice/modernisation-platform-environments" } - ) - - environment = trimprefix(terraform.workspace, "${var.networking[0].application}-") - vpc_name = var.networking[0].business-unit - subnet_set = var.networking[0].set - vpc_all = "${local.vpc_name}-${local.environment}" - subnet_set_name = "${var.networking[0].business-unit}-${local.environment}-${var.networking[0].set}" - - is_live = [substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-production" || substr(terraform.workspace, length(local.application_name), length(terraform.workspace)) == "-preproduction" ? "live" : "non-live"] - provider_name = "core-vpc-${local.environment}" - - # environment specfic variables - # example usage: - # example_data = local.application_data.accounts[local.environment].example_var - application_data = fileexists("./application_variables.json") ? jsondecode(file("./application_variables.json")) : null -} diff --git a/terraform/environments/data-platform/platform_providers.tf b/terraform/environments/data-platform/platform_providers.tf deleted file mode 100644 index 828f987c7aa..00000000000 --- a/terraform/environments/data-platform/platform_providers.tf +++ /dev/null @@ -1,58 +0,0 @@ -# AWS provider for the original session which you connect with -provider "aws" { - alias = "original-session" - region = "eu-west-2" -} - -# AWS provider for the workspace you're working in (every resource will default to using this, unless otherwise specified) -provider "aws" { - region = "eu-west-2" - assume_role { - role_arn = !can(regex("githubactionsrolesession|AdministratorAccess|user", data.aws_caller_identity.original_session.arn)) ? null : can(regex("user", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids[terraform.workspace]}:role/${var.collaborator_access}" : "arn:aws:iam::${data.aws_caller_identity.original_session.id}:role/MemberInfrastructureAccess" - } -} - -# AWS provider for the Modernisation Platform, to get things from there if required -provider "aws" { - alias = "modernisation-platform" - region = "eu-west-2" - assume_role { - role_arn = "arn:aws:iam::${local.modernisation_platform_account_id}:role/modernisation-account-limited-read-member-access" - } -} - -# AWS provider for core-vpc-, to access resources in the core-vpc accounts -provider "aws" { - alias = "core-vpc" - region = "eu-west-2" - assume_role { - role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids[local.provider_name]}:role/member-delegation-read-only" : "arn:aws:iam::${local.environment_management.account_ids[local.provider_name]}:role/member-delegation-${local.vpc_name}-${local.environment}" - } -} - -# AWS provider for network services to enable dns entries for certificate validation to be created -provider "aws" { - alias = "core-network-services" - region = "eu-west-2" - assume_role { - role_arn = !can(regex("githubactionsrolesession|AdministratorAccess", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/read-log-records" : "arn:aws:iam::${local.environment_management.account_ids["core-network-services-production"]}:role/modify-dns-records" - } -} - -# Provider for creating resources in us-east-1, eg ACM resources for CloudFront -provider "aws" { - alias = "us-east-1" - region = "us-east-1" - assume_role { - role_arn = !can(regex("githubactionsrolesession|AdministratorAccess|user", data.aws_caller_identity.original_session.arn)) ? null : can(regex("user", data.aws_caller_identity.original_session.arn)) ? "arn:aws:iam::${local.environment_management.account_ids[terraform.workspace]}:role/${var.collaborator_access}" : "arn:aws:iam::${data.aws_caller_identity.original_session.id}:role/MemberInfrastructureAccessUSEast" - } -} - -# Provider for reading resources from root account IdentityStore -provider "aws" { - region = "eu-west-2" - alias = "sso-readonly" - assume_role { - role_arn = "arn:aws:iam::${local.environment_management.aws_organizations_root_account_id}:role/ModernisationPlatformSSOReadOnly" - } -} diff --git a/terraform/environments/data-platform/platform_secrets.tf b/terraform/environments/data-platform/platform_secrets.tf deleted file mode 100644 index bb006856534..00000000000 --- a/terraform/environments/data-platform/platform_secrets.tf +++ /dev/null @@ -1,17 +0,0 @@ -# Get modernisation account id from ssm parameter -data "aws_ssm_parameter" "modernisation_platform_account_id" { - provider = aws.original-session - name = "modernisation_platform_account_id" -} - -# Get secret by arn for environment management -data "aws_secretsmanager_secret" "environment_management" { - provider = aws.modernisation-platform - name = "environment_management" -} - -# Get latest secret value with ID from above. This secret stores account IDs for the Modernisation Platform sub-accounts -data "aws_secretsmanager_secret_version" "environment_management" { - provider = aws.modernisation-platform - secret_id = data.aws_secretsmanager_secret.environment_management.id -} diff --git a/terraform/environments/data-platform/platform_versions.tf b/terraform/environments/data-platform/platform_versions.tf deleted file mode 100644 index 6161ef3bc02..00000000000 --- a/terraform/environments/data-platform/platform_versions.tf +++ /dev/null @@ -1,13 +0,0 @@ -terraform { - required_providers { - aws = { - version = "~> 5.0" - source = "hashicorp/aws" - } - http = { - version = "~> 3.0" - source = "hashicorp/http" - } - } - required_version = "~> 1.0" -} diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf deleted file mode 100644 index a3b89b8e609..00000000000 --- a/terraform/environments/data-platform/s3.tf +++ /dev/null @@ -1,276 +0,0 @@ -module "data_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" - - bucket_prefix = "data-${local.environment}" - versioning_enabled = false - # Refer to the below section "Replication" before enabling replication - replication_enabled = false - bucket_policy = [data.aws_iam_policy_document.data_s3_bucket_policy_document.json] - providers = { - # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the - # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation - # Platform team to add a new provider for the additional Region. - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 730 - } - } - ] - - tags = local.tags -} - - -module "metadata_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" - - bucket_prefix = "metadata-${local.environment}" - versioning_enabled = false - # Refer to the below section "Replication" before enabling replication - replication_enabled = false - bucket_policy = [data.aws_iam_policy_document.metadata_s3_bucket_policy_document.json] - providers = { - # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the - # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation - # Platform team to add a new provider for the additional Region. - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 730 - } - - } - ] - - tags = local.tags -} - - -module "logs_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" - - bucket_prefix = "logs-${local.environment}" - versioning_enabled = false - # Refer to the below section "Replication" before enabling replication - replication_enabled = false - bucket_policy = [data.aws_iam_policy_document.logs_s3_bucket_policy_document.json] - providers = { - # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the - # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation - # Platform team to add a new provider for the additional Region. - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 730 - } - - } - ] - - tags = local.tags -} - - - -module "data_landing_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" - - bucket_prefix = "data-landing-${local.environment}" - versioning_enabled = false - # Refer to the below section "Replication" before enabling replication - replication_enabled = false - bucket_policy = [data.aws_iam_policy_document.data_landing_s3_bucket_policy_document.json] - providers = { - # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the - # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation - # Platform team to add a new provider for the additional Region. - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 730 - } - - } - ] - - tags = local.tags -} - -module "s3_athena_query_results_bucket" { #tfsec:ignore:aws-s3-enable-versioning - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0" - - bucket_name = "athena-data-product-query-results-${data.aws_caller_identity.current.account_id}" - versioning_enabled = false - # Refer to the below section "Replication" before enabling replication - replication_enabled = false - force_destroy = true - - providers = { - # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the - # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation - # Platform team to add a new provider for the additional Region. - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 30 - storage_class = "STANDARD_IA" - }, { - days = 90 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 365 - } - - noncurrent_version_transition = [ - { - days = 30 - storage_class = "STANDARD_IA" - }, { - days = 90 - storage_class = "GLACIER" - } - ] - - noncurrent_version_expiration = { - days = 365 - } - } - ] - - tags = local.tags -} - -resource "aws_s3_bucket_notification" "bucket_notification" { - bucket = module.data_s3_bucket.bucket.id - eventbridge = true -} - -resource "aws_s3_bucket_notification" "landing_bucket_notification" { - bucket = module.data_landing_s3_bucket.bucket.id - eventbridge = true -} - -# load the json schema for data product metadata -resource "aws_s3_object" "object_metadata" { - bucket = module.metadata_s3_bucket.bucket.id - key = "data_product_metadata_spec/v1.1.0/moj_data_product_metadata_spec.json" - source = "data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json" - etag = filemd5("data-product-metadata-json-schema/v1.1.0/moj_data_product_metadata_spec.json") - acl = "bucket-owner-full-control" - server_side_encryption = "AES256" -} - -# load the json schema for data product schema -resource "aws_s3_object" "object_schema" { - bucket = module.metadata_s3_bucket.bucket.id - key = "data_product_schema_spec/v1.0.0/moj_data_product_schema_spec.json" - source = "data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json" - etag = filemd5("data-product-table-schema-json-schema/v1.0.0/moj_data_product_table_spec.json") - acl = "bucket-owner-full-control" - server_side_encryption = "AES256" -} diff --git a/terraform/environments/data-platform/secrets.tf b/terraform/environments/data-platform/secrets.tf deleted file mode 100644 index 3c904826b94..00000000000 --- a/terraform/environments/data-platform/secrets.tf +++ /dev/null @@ -1,16 +0,0 @@ - -# API auth secret (this should be a key-value secret with an api-auth-token key) -resource "aws_secretsmanager_secret" "api_auth" { - name = "data-platform-api-auth-token" - tags = local.tags -} - -data "aws_secretsmanager_secret_version" "api_auth" { - secret_id = aws_secretsmanager_secret.api_auth.id -} - -# openmeta jwt for api -resource "aws_secretsmanager_secret" "openmetadata" { - name = "data-platform-openmetadata-token" - tags = local.tags -} diff --git a/terraform/environments/data-platform/tests/delete_table.py b/terraform/environments/data-platform/tests/delete_table.py deleted file mode 100644 index 28f6858e502..00000000000 --- a/terraform/environments/data-platform/tests/delete_table.py +++ /dev/null @@ -1,37 +0,0 @@ -import json -import os -import sys - -import boto3 -import requests - - -data_product_name = "example_prison_data_product" -table_name = "testing" -base_url = "https://hsolkci589.execute-api.eu-west-2.amazonaws.com/development" -schema_url = f"/data-product/{data_product_name}/table/{table_name}" -url = base_url + schema_url -glue = boto3.client("glue") - -try: - auth_token = json.loads(os.environ["API_AUTH"]) - auth_token = auth_token["auth-token"] -except KeyError: - print("API_AUTH environment variable should be set to a json containing auth-token") - sys.exit(1) - -headers = {"authorizationToken": auth_token} - -# Delete schema request -response = requests.delete( - url=url, - headers=headers, -) - -if response.status_code != 200: - print(f"Error deleting data product schema. Status code: {response.status_code}") - print(f"Error deleting data product schema. Response: {response.text}") - print("Exiting...") - sys.exit(1) - -response_json = response.json() diff --git a/terraform/environments/data-platform/tests/requirements.txt b/terraform/environments/data-platform/tests/requirements.txt deleted file mode 100644 index fafb8403733..00000000000 --- a/terraform/environments/data-platform/tests/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests==2.31.0 -boto3==1.28.51 diff --git a/terraform/environments/data-platform/tests/run_smoke_tests.py b/terraform/environments/data-platform/tests/run_smoke_tests.py deleted file mode 100644 index 2bbfebf1469..00000000000 --- a/terraform/environments/data-platform/tests/run_smoke_tests.py +++ /dev/null @@ -1,164 +0,0 @@ -import base64 -import hashlib -import json -import os -import sys -import time -from datetime import datetime, timedelta, timezone - -import requests - -filename = "test_data.csv" -data_product_name = "example_prison_data_product" -table_name = "testing" -base_url = "https://hsolkci589.execute-api.eu-west-2.amazonaws.com/development" - -try: - auth_token = json.loads(os.environ["API_AUTH"]) - auth_token = auth_token["auth-token"] -except KeyError: - print("API_AUTH environment variable should be set to a json containing auth-token") - sys.exit(1) - - -def md5_hash_file_contents(file) -> str: - """This holds the file in memory to hash so - will be unsuitable for large files""" - with open(file, "r") as tempfile: - body = tempfile.read() - md = hashlib.md5(body.encode("utf-8")).digest() - contents_md5 = base64.b64encode(md).decode("utf-8") - - return contents_md5 - -class APIClient: - def __init__(self, base_url, auth_token): - self.table_url = base_url + f"/data-product/{data_product_name}/table/{table_name}" - self.data_product_url = base_url + f"/data-product/{data_product_name}" - self.register_url = base_url + "/data-product/register" - self.preview_data_url = self.table_url + "/preview" - self.presigned_url = base_url + f"/data-product/{data_product_name}/table/{table_name}/upload" - self.headers = {"authorizationToken": auth_token} - - def register(self): - print("Registering data product...") - - metadata = { - "name": "example_prison_data_product", - "description": "just testing the metadata json validation/registration", - "domain": "MoJ", - "dataProductOwner": "matthew.laverty@justice.gov.uk", - "dataProductOwnerDisplayName": "matt laverty", - "email": "matthew.laverty@justice.gov.uk", - "status": "draft", - "dpiaRequired": False, - "retentionPeriod": 3650 - } - - return requests.post( - url=self.register_url, - headers=self.headers, - json={ - "metadata": metadata - } - ) - - def delete_data_product(self): - print("Deleting data product...") - return requests.delete( - url=self.data_product_url, - headers=self.headers, - ) - - def preview_data(self): - print("Fetching data") - return requests.get( - url=self.preview_data_url, - headers=self.headers, - ) - - def create_schema(self): - print("Creating a schema...") - return requests.post( - url=self.table_url + "/schema", - headers=self.headers, - json={ - "schema": { - "tableDescription": "just a test table", - "columns": [ - {"name": "col_0", "type": "double", "description": "just a test column"}, - {"name": "col_1", "type": "double", "description": "just a test column"}, - {"name": "col_2", "type": "double", "description": "just a test column"}, - {"name": "col_3", "type": "double", "description": "just a test column"}, - ] - } - } - ) - - def upload_file(self): - file_md5_hash = md5_hash_file_contents(filename) - - body = { - "filename": filename, - "contentMD5": file_md5_hash, - } - - # Get presigned url - response = requests.post( - url=self.presigned_url, - json=body, - headers=self.headers, - ) - - if response.status_code != 200: - print(f"Error getting presigned url. Status code: {response.status_code}") - print(f"Error getting presigned url. Response: {response.text}") - raise Exception('Error getting presigned URL') - - response_json = response.json() - post_policy_form_data = response_json["URL"]["fields"] - multipart_form_data = { - **post_policy_form_data, - "file": (post_policy_form_data["key"], open(filename, "r")), - } - - # Upload data - print("Uploading data") - return requests.post(response_json["URL"]["url"], files=multipart_form_data) - - -def parse_first_line_of_data(output): - lines = output.splitlines() - fields = [i.strip() for i in lines[1].split("|")] - row = fields[1:-1] - if len(row) != 5: - raise ValueError(row) - - col1, col2, col3, col4, extraction_timestamp = fields[1:-1] - - parsed_datetime = datetime.strptime(extraction_timestamp, r'%Y%m%dT%H%M%SZ') - age = parsed_datetime - datetime.now() - - return col1, col2, col3, col4, age - - -def run_test(client): - upload_response = client.upload_file() - print(upload_response.status_code, upload_response.text) - print(f"Waiting for {data_product_name}.{table_name} to create in athena") - - time.sleep(10) - - preview_repsonse = client.preview_data() - if preview_repsonse.status_code != 200: - print(f"Error previewing data: {preview_repsonse.status_code} {preview_repsonse.text}") - - print(preview_repsonse.text) - col1, col2, col3, col4, age = parse_first_line_of_data(preview_repsonse.text) - - assert (col1, col2, col3, col4) == ("0.1915194503788923", "0.3648859839013723", "0.0598092227798519", "0.2852509600245098") - assert age < timedelta(seconds=15) - - -client = APIClient(base_url, auth_token) -run_test(client) \ No newline at end of file diff --git a/terraform/environments/data-platform/tests/test_data.csv b/terraform/environments/data-platform/tests/test_data.csv deleted file mode 100644 index eaf30d8eb1c..00000000000 --- a/terraform/environments/data-platform/tests/test_data.csv +++ /dev/null @@ -1,21 +0,0 @@ -col_0,col_1,col_2,col_3 -0.1915194503788923,0.3648859839013723,0.0598092227798519,0.2852509600245098 -0.6221087710398319,0.6153961784334937,0.18428708381381365,0.624916705305911 -0.4377277390071145,0.07538124164297655,0.04735527880151513,0.47809379567067456 -0.7853585837137692,0.3688240060019745,0.6748809435823302,0.19567517866589823 -0.7799758081188035,0.9331401019825216,0.5946247799344488,0.38231745203150647 -0.2725926052826416,0.6513781432265774,0.5333101629987506,0.05387368514623658 -0.2764642551430967,0.3972025777261542,0.04332406269480349,0.45164840826085906 -0.8018721775350193,0.7887301429407455,0.5614330800633979,0.9820047415219545 -0.9581393536837052,0.31683612216887125,0.329668445620915,0.12394270048696299 -0.8759326347420947,0.5680986526260692,0.5029668331126184,0.1193808979262484 -0.35781726995786667,0.8691273895612258,0.11189431757440382,0.7385230561433468 -0.5009951255234587,0.43617342389567937,0.6071937062184846,0.5873036334639846 -0.6834629351721363,0.8021476420801591,0.5659446430505314,0.4716325343203678 -0.7127020269829002,0.14376682451456457,0.00676406199000279,0.1071268171938663 -0.37025075479039493,0.7042609711183354,0.617441708804297,0.22921856546061792 -0.5611961860656249,0.7045813081895725,0.9121228864331543,0.8999651948366754 -0.5030831653078097,0.21879210567408858,0.7905241330570334,0.4167535378026932 -0.013768449590682241,0.924867628615565,0.9920814661883615,0.5358516625316159 -0.772826621612374,0.44214075540417663,0.9588017621528665,0.006208516587129398 -0.8826411906361166,0.9093159589724725,0.7919641352916398,0.3006417057703011 diff --git a/terraform/environments/data-platform/triggers.tf b/terraform/environments/data-platform/triggers.tf deleted file mode 100644 index 883107f6d39..00000000000 --- a/terraform/environments/data-platform/triggers.tf +++ /dev/null @@ -1,42 +0,0 @@ - -resource "aws_cloudwatch_event_rule" "object_created_raw_data" { - name = "object_created_raw_data" - tags = local.tags - event_pattern = jsonencode({ - "source" : ["aws.s3"], - "detail-type" : ["Object Created"], - "detail" : { - "bucket" : { "name" : [module.data_s3_bucket.bucket.id] }, - "object" : { - "key" : [{ "prefix" : "raw/" }] - } - } - }) -} - -resource "aws_cloudwatch_event_rule" "object_created_data_landing" { - name = "object_created_data_landing" - tags = local.tags - event_pattern = jsonencode({ - "source" : ["aws.s3"], - "detail-type" : ["Object Created"], - "detail" : { - "bucket" : { "name" : [module.data_landing_s3_bucket.bucket.id] }, - "object" : { - "key" : [{ "prefix" : "landing/" }] - } - } - }) -} - -resource "aws_cloudwatch_event_target" "athena_load_lambda_trigger" { - rule = aws_cloudwatch_event_rule.object_created_raw_data.name - target_id = "athena" - arn = module.data_product_athena_load_lambda.lambda_function_arn -} - -resource "aws_cloudwatch_event_target" "object_created_data_landing" { - rule = aws_cloudwatch_event_rule.object_created_data_landing.name - target_id = "landing_to_raw" - arn = module.data_product_landing_to_raw_lambda.lambda_function_arn -} diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf deleted file mode 100644 index db0f3160a90..00000000000 --- a/terraform/environments/data-platform/variables.tf +++ /dev/null @@ -1,63 +0,0 @@ -variable "docs_versions" { - type = map(any) -} - -variable "authorizer_versions" { - type = map(any) -} - -variable "presigned_url_versions" { - type = map(any) -} - -variable "athena_load_versions" { - type = map(any) -} - -variable "create_metadata_versions" { - type = map(any) -} - -variable "resync_unprocessed_files_versions" { - type = map(any) -} - -variable "reload_data_product_versions" { - type = map(any) -} - -variable "landing_to_raw_versions" { - type = map(any) -} - -variable "create_schema_versions" { - type = map(any) -} - -variable "get_schema_versions" { - type = map(any) -} - -variable "update_metadata_versions" { - type = map(any) -} - -variable "update_schema_versions" { - type = map(any) -} - -variable "preview_data_versions" { - type = map(any) -} - -variable "delete_table_for_data_product_versions" { - type = map(any) -} - -variable "push_to_catalogue_versions" { - type = map(any) -} - -variable "delete_data_product_versions" { - type = map(any) -} diff --git a/terraform/environments/data-platform/waf.tf b/terraform/environments/data-platform/waf.tf deleted file mode 100644 index 907c72b5360..00000000000 --- a/terraform/environments/data-platform/waf.tf +++ /dev/null @@ -1,66 +0,0 @@ -resource "aws_wafv2_web_acl" "waf" { - name = local.environment - scope = "REGIONAL" - - default_action { - allow {} - } - - rule { - name = "AWS-AWSManagedRulesCommonRuleSet" - priority = 1 - - override_action { - count {} - } - - statement { - managed_rule_group_statement { - name = "AWSManagedRulesCommonRuleSet" - vendor_name = "AWS" - } - } - - visibility_config { - cloudwatch_metrics_enabled = true - metric_name = "AWSManagedRulesCommonRuleSet" - sampled_requests_enabled = false - } - } - - rule { - name = "AWS-AWSManagedRulesAmazonIpReputationList" - priority = 2 - - override_action { - count {} - } - - statement { - managed_rule_group_statement { - name = "AWSManagedRulesAmazonIpReputationList" - vendor_name = "AWS" - } - } - - visibility_config { - cloudwatch_metrics_enabled = true - metric_name = "AWSManagedRulesAmazonIpReputationList" - sampled_requests_enabled = false - } - } - - - visibility_config { - cloudwatch_metrics_enabled = true - metric_name = "waf" - sampled_requests_enabled = false - } - - tags = local.tags -} - -resource "aws_wafv2_web_acl_association" "association" { - resource_arn = aws_api_gateway_stage.default_stage.arn - web_acl_arn = aws_wafv2_web_acl.waf.arn -}