Merge pull request #56 from NASA-PDS/terraform-deploy

Deployment of registry-api and nucleus completed

.github/CODEOWNERS

@@ -0,0 +1,46 @@
+# 📀 Code Owners
+#
+# Copyright © 2021, California Institute of Technology ("Caltech").
+# U.S. Government sponsorship acknowledged.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# • Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# • Redistributions must reproduce the above copyright notice, this list of
+#   conditions and the following disclaimer in the documentation and/or other
+#   materials provided with the distribution.
+# • Neither the name of Caltech nor its operating division, the Jet Propulsion
+#   Laboratory, nor the names of its contributors may be used to endorse or
+#   promote products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+#
+# Each line is a file pattern followed by one or more owners.
+#
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# @global-owner1 and @global-owner2 will be requested for
+# review when someone opens a pull request.
+#
+# Go to https://github.com/orgs/NASA-PDS/teams to find out about our teams
+
+* @tloubrieu-jpl @nutjob4life @sjoshi-jpl @ramesh-maddegoda @jordanpadams
+
+# For more information on populating this file, check out
+# https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners

.github/workflows/terraform-cicd.yaml

@@ -0,0 +1,86 @@
+name: 'Terraform-Workflow'
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    types: [opened, edited, reopened, ready_for_review, review_requested]
+  workflow_dispatch:
+
+permissions:
+  id-token: write  # required to use OIDC authentication
+  contents: read # required to checkout the code from the repo
+
+jobs:
+  deploy:
+    name: 'Terraform'
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./terraform
+    environment: NGAP DEV
+    env:
+      TF_LOG: INFO
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v2
+      with:
+        aws-region: "${{ secrets.AWS_REGION }}"
+        role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
+        role-session-name: PDS_GitHub_OIDC_TERRAFORM
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v2
+
+    - name: Terraform Fmt, Init & Validate
+      id: fmt_init_validate
+      env:
+        AWS_BUCKET_NAME: ${{ secrets.AWS_BUCKET_NAME}}
+        AWS_BUCKET_KEY_NAME: ${{ secrets.AWS_BUCKET_KEY_NAME}}
+      run: |
+        terraform fmt
+        terraform init -backend-config="bucket=${AWS_BUCKET_NAME}" -backend-config="key=${AWS_BUCKET_KEY_NAME}"
+        terraform validate -no-color
+
+    - name: Terraform Plan
+      id: plan
+      if: github.event_name == 'push' || github.event_name == 'pull_request'
+      env:
+        TF_VAR_efs_file_system_id: "${{ secrets.EFS_FILE_SYSTEM_ID }}"
+        TF_VAR_registry_loader_scripts_access_point_id: "${{ secrets.REGISTRY_LOADER_SCRIPTS_ACCESS_POINT_ID }}"
+        TF_VAR_registry_loader_default_configs_access_point_id: "${{ secrets.REGISTRY_LOADER_DEFAULT_CONFIGS_ACCESS_POINT_ID }}"
+        TF_VAR_task_role_arn: "${{ secrets.TASK_ROLE_ARN }}"
+        TF_VAR_execution_role_arn: "${{ secrets.EXECUTION_ROLE_ARN }}"
+        TF_VAR_vpc_id: "${{ secrets.VPC_ID }}"
+        TF_VAR_vpc_cidr: "${{ secrets.VPC_CIDR }}"
+        TF_VAR_nucleus_security_group_ingress_cidr: "${{ secrets.NUCLEUS_SG_INGRESS_CIDR }}"
+        TF_VAR_subnet_ids: "${{ secrets.SUBNET_IDS }}"
+        TF_VAR_airflow_execution_role: "${{ secrets.AIRFLOW_EXECUTION_ROLE }}"
+      run: terraform plan -no-color
+      continue-on-error:  true
+
+    - name: Terraform Plan Status
+      id : plan_status
+      if: steps.plan.outcome == 'failure'
+      run: exit 1
+
+    - name: Terraform Apply
+      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      env:
+        TF_VAR_efs_file_system_id: "${{ secrets.EFS_FILE_SYSTEM_ID }}"
+        TF_VAR_registry_loader_scripts_access_point_id: "${{ secrets.REGISTRY_LOADER_SCRIPTS_ACCESS_POINT_ID }}"
+        TF_VAR_registry_loader_default_configs_access_point_id: "${{ secrets.REGISTRY_LOADER_DEFAULT_CONFIGS_ACCESS_POINT_ID }}"
+        TF_VAR_task_role_arn: "${{ secrets.TASK_ROLE_ARN }}"
+        TF_VAR_execution_role_arn: "${{ secrets.EXECUTION_ROLE_ARN }}"
+        TF_VAR_vpc_id: "${{ secrets.VPC_ID }}"
+        TF_VAR_vpc_cidr: "${{ secrets.VPC_CIDR }}"
+        TF_VAR_nucleus_security_group_ingress_cidr: "${{ secrets.NUCLEUS_SG_INGRESS_CIDR }}"
+        TF_VAR_subnet_ids: "${{ secrets.SUBNET_IDS }}"
+        TF_VAR_airflow_execution_role: "${{ secrets.AIRFLOW_EXECUTION_ROLE }}"
+      run: terraform apply -auto-approve

.gitignore

@@ -0,0 +1,126 @@
+# Copyright © 2021, California Institute of Technology ("Caltech").
+# U.S. Government sponsorship acknowledged.
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# • Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# • Redistributions must reproduce the above copyright notice, this list of
+#   conditions and the following disclaimer in the documentation and/or other
+#   materials provided with the distribution.
+# • Neither the name of Caltech nor its operating division, the Jet Propulsion
+#   Laboratory, nor the names of its contributors may be used to endorse or
+#   promote products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# Items for Git to ignore
+
+# Python build, virtual environments, and buildouts
+.venv
+venv
+__pycache__/
+dist/
+build/
+*.egg-info
+.*.cfg
+develop-eggs/
+.python-eggs/
+.eggs/
+pip-selfcheck.json
+.python-version
+
+# Python testing artifacts
+.coverage
+htmlcov
+.tox/
+
+# Object files
+*.o
+*.pkl
+*.py[ocd]
+
+# Libraries
+*.lib
+*.a
+
+# Eclipse files
+.settings/
+*.project
+*.classpath
+
+# Editor support
+.idea/
+*.iml
+.vscode
+*.sublime-*
+
+# Shared objects (inc. Windows DLLs)
+*.dll
+*.so
+*.so.*
+*.dylib
+lib/
+lib64/
+
+# Executables
+*.exe
+*.out
+*.app
+
+# Temporary files
+*~
+.*.swp
+var/
+
+# other stuff
+*.log
+*.xpr
+bin/
+.*.swp
+typescript
+
+# OS-specific artifacts
+.DS_Store
+._*
+
+# Exclusions
+!.coveragerc
+!.editorconfig
+!.gitattributes
+!.gitignore
+!.gitkeep
+
+# Local .terraform state files, directories and crash logs
+**/.terraform/*
+*.tfstate
+*.tfstate.*
+*.tfvars
+*.tfvars.json
+crash.log
+crash.*.log
+
+# Ignore Terraform CLI configuration files
+.terraformrc
+terraform.rc
+
+# Ignore Terraform override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json

.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+    - id: trailing-whitespace
+    - id: end-of-file-fixer
+    - id: check-merge-conflict
+    - id: debug-statements
+    - id: check-yaml
+    - id: check-json #checks json files for parseable syntax.
+    - id: pretty-format-json #sets a standard for formatting json files.
+
+-   repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.81.0
+    hooks:
+    - id: terraform_fmt #Rewrites all Terraform configuration files to a canonical format.
+    - id: terraform_validate #Validates all Terraform configuration files.
+    - id: terraform_tflint #Validates all Terraform configuration files with TFLint.
+    # - id: terrascan #Detect compliance and security violations of Terraform templates.
+    # - id: terraform_tfsec
+    # - id: terraform_docs

terraform/README.md

@@ -11,8 +11,8 @@ Therefore, as a result of the Terraform scripts in this directory following thin
 
 
 Note: In addition to the above components, there are Terraform modules, container definitions and a DAG file
-included to deploy PDS Registry related ECS tasks, a DAG and an EFS file system that can be used to demonstrate 
-an example PDS Registry use case. However, these additional components are not part of the MVP of 
+included to deploy PDS Registry related ECS tasks, a DAG and an EFS file system that can be used to demonstrate
+an example PDS Registry use case. However, these additional components are not part of the MVP of
 PDS Nucleus data pipeline. These PDS Registry related terraform modules are still under development (not part of the PDS Nucleus Baseline Deployment task)
 and are kept disabled in the main.tf terraform file.
 
@@ -30,7 +30,7 @@ and are kept disabled in the main.tf terraform file.
 2. Ability to get AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN for the AWS account
 
 3. Terraform is installed in local environment (This was tested with Terraform v1.3.7. Any higher version should also work)
- - Instructions to install Terraform is available at https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli 
+ - Instructions to install Terraform is available at https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli
 
 4. A VPC and one or more subnets should be available on AWS (obtain the VPC ID and subnet IDs from AWS console or from the AWS
 system admin team of your AWS account)
@@ -56,16 +56,16 @@ cd nucleus/terraform
     - AWS_SESSION_TOKEN
     - AWS_DEFAULT_REGION
 
-4. Open the `variables.tf` file at `nucleus/terraform/terraform-modules/mwaa-env/variables.tf` and
-update the following variables to match with your AWS Setup. Most of the below values can be obtained by
-the system admin team of your AWS account.
+4. Create a `terraform.tfvars` file locally and enter the value for variables specified in `variables.tf` file at `nucleus/terraform/terraform-modules/mwaa-env/variables.tf`. Ensure these values match with your AWS Setup. Most of the below values can be obtained by the system admin team of your AWS account.
 
     - vpc_id:  VPC ID of your AWS VPC
     - vpc_cidr: VPC CIDR for MWAA (E.g.: "10.1.0.0/16")
     - nucleus_security_group_ingress_cidr: List of ingress CIDRs for the Nucleus Security Group to be created (E.g.: "10.21.240.0/20")
     - subnet_ids: List of Subnet IDs to be used for the MWAA
     - airflow_execution_role: Airflow AWS Execution Role
 
+> Note: `terraform.tfvars` is only used to test with your configuration with the actual values in your AWS account. This file will not be uploaded to GitHub as it's ignored by Git. Once testing is completed successfully work with your admin to get the values for these tested variables updated via GitHub secrets, which are dynamically passed in during runtime.
+
 5. Initialize Terraform working directory.
 
 ```shell

terraform/main.tf

@@ -1,17 +1,29 @@
 # The Terraform module to create the PDS Nucleus Baseline System (without any project specific components)
 module "mwaa-env" {
   source = "./terraform-modules/mwaa-env"
+
+  vpc_id                              = var.vpc_id
+  vpc_cidr                            = var.vpc_cidr
+  nucleus_security_group_ingress_cidr = var.nucleus_security_group_ingress_cidr
+  subnet_ids                          = var.subnet_ids
+  airflow_execution_role              = var.airflow_execution_role
 }
 
 
 # The following modules are specific to PDS Registry and are under development. These modules are currently
 # capable of successfully deploying some ECS tasks related with PDS Registry. However, these modules
 # are currently disabled to keep the PDS Nucleus Baseline System clean and to avoid confusions.
 
-#module "efs" {
-#  source = "./terraform-modules/efs"
-#}
-#
-#module "ecs" {
-#  source = "./terraform-modules/ecs"
-#}
+# module "efs" {
+#   source = "./terraform-modules/efs"
+# }
+
+module "ecs" {
+  source = "./terraform-modules/ecs"
+
+  efs_file_system_id                              = var.efs_file_system_id
+  registry_loader_scripts_access_point_id         = var.registry_loader_scripts_access_point_id
+  registry_loader_default_configs_access_point_id = var.registry_loader_default_configs_access_point_id
+  task_role_arn                                   = var.task_role_arn
+  execution_role_arn                              = var.execution_role_arn
+}

terraform/providers.tf

@@ -1,4 +1,5 @@
 terraform {
+  required_version = ">= 0.14.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"