Merge pull request #434 from tmobile/develop

Release - v1.11.1

installscripts/cookbooks/Policyfile.rb

@@ -4,4 +4,4 @@
 cookbook 'nodejs', '~> 5.0.0'
 cookbook 'cloudcli', '~> 1.2.0'
 cookbook 'jenkins', path: './jenkins'
-run_list 'jenkins::setupjenkins', 'jenkins::sonarqube-scanner', 'jenkins::configurejenkins'
+run_list 'jenkins::setupjenkins', 'jenkins::sonarqube-scanner', 'jenkins::configurejenkins', 'jenkins::installgolang'

installscripts/cookbooks/jenkins/attributes/default.rb

@@ -13,6 +13,7 @@
 default['scmpath'] = "#{node['scmelb']}/scm"
 default['region'] = 'us-east-1'
 default['git_branch'] = 'master'
+default['git_plugin_branch'] = 'master'
 default['git_repo'] = 'https://github.com/tmobile/jazz.git'
 default['git_content_repo'] = 'https://github.com/tmobile/jazz-content/raw'
 default['git_content_plugins'] = 'jenkins/staticplugins/plugins.tar'

installscripts/cookbooks/jenkins/files/default/jazz-installer-vars.json

@@ -8,7 +8,8 @@
         "CLOUDFRONT_ORIGIN_ID": "",
         "ACCOUNTID": "",
         "REGION": "",
-        "ROLEID": "",
+        "PLATFORMSERVICES_ROLEID": "",
+        "USERSERVICES_ROLEID": "",
         "ES_HOSTNAME": "",
         "COGNITO": {
             "USER_POOL_ID": "",

installscripts/cookbooks/jenkins/recipes/configurejenkins.rb

@@ -44,7 +44,7 @@
 
 # Fetch the xmls.tar from our content repo
 execute 'copyXmlsTar' do
-  command "curl -sL #{node['git_content_repo']}/#{node['git_branch']}/#{node['git_content_xmls']} -o #{node['chef_root']}/xmls.tar; chmod 755 #{node['chef_root']}/xmls.tar"
+  command "curl -sL #{node['git_content_repo']}/#{node['git_plugin_branch']}/#{node['git_content_xmls']} -o #{node['chef_root']}/xmls.tar; chmod 755 #{node['chef_root']}/xmls.tar"
 end
 
 #ToDo ChefRemoval

installscripts/cookbooks/jenkins/recipes/installgolang.rb

@@ -0,0 +1,40 @@
+if node['dockerizedJenkins'] == false
+    # Installing go lang
+    remote_file "#{Chef::Config['file_cache_path']}/go1.10.3.linux-amd64.tar.gz" do
+        source 'https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz'
+        mode '0755'
+        action :create
+    end
+
+    execute 'mkdir' do
+        command 'mkdir -p /opt/go/{bin,src,pkg}'
+    end
+
+    execute 'install_go' do
+        command "tar -C /usr/local -xzf #{Chef::Config['file_cache_path']}/go1.10.3.linux-amd64.tar.gz"
+    end
+
+    # Installing godep (Dependency Management tool)
+    remote_file "#{Chef::Config['file_cache_path']}/install.sh" do
+        source 'https://raw.githubusercontent.com/golang/dep/master/install.sh'
+        mode '0755'
+        action :create
+    end
+
+    execute 'install_godep' do
+        environment ({
+            'GOPATH' => "/opt/go",
+            'PATH' => "#{ENV['PATH']}:"+"#{ENV['GOPATH']}"+"/bin:"+"/usr/local/go/bin/"
+        })
+        command "bash #{Chef::Config['file_cache_path']}/install.sh"
+    end
+    # link go
+    link '/bin/go' do
+        to '/usr/local/go/bin/go'
+    end
+    # link dep
+    link '/bin/dep' do
+        to '/opt/go/bin/dep'
+    end
+end
+

installscripts/cookbooks/jenkins/recipes/setupjenkins.rb

@@ -19,7 +19,7 @@
   # plugin management
   # Fetch the plugins.tar from our content repo TODO replace this with a dynamic plugin install like the dockerized version
   execute 'copyPluginsTar' do
-    command "curl -sL #{node['git_content_repo']}/#{node['git_branch']}/#{node['git_content_repo']} -o #{node['chef_root']}/plugins.tar; chmod 755 #{node['chef_root']}/plugins.tar"
+    command "curl -sL #{node['git_content_repo']}/#{node['git_plugin_branch']}/#{node['git_content_plugins']} -o #{node['chef_root']}/plugins.tar; chmod 755 #{node['chef_root']}/plugins.tar"
   end
 
   execute 'extractJenkinsPlugins' do

installscripts/dockerfiles/jenkins-ce/Dockerfile

@@ -2,12 +2,21 @@
 FROM jenkins/jenkins:2.121.3
 MAINTAINER JazzOSS Team
 
+ARG goVersion=1.10.3
 # Switching to root to configure the image with system packages
 USER root
 RUN apt-get update && apt-get install -y vim curl sudo libtool autoconf make unzip rsync gcc autogen shtool pkg-config lsb-release python python-dev python-pip python-setuptools groff less && \
 curl https://bootstrap.pypa.io/get-pip.py | python && pip install --upgrade awscli && apt-get clean && pip install virtualenv && /usr/bin/easy_install virtualenv
 RUN wget -O /opt/apache-maven-3.5.2-bin.tar.gz https://archive.apache.org/dist/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz && tar xzvf /opt/apache-maven-3.5.2-bin.tar.gz -C  /opt && export PATH=$PATH:/opt/apache-maven-3.5.2/bin >> /etc/profile.d/maven.sh && ln -sf /opt/apache-maven-3.5.2/bin/mvn /usr/bin/mvn
 RUN curl -sL https://deb.nodesource.com/setup_8.x | bash && apt-get install -y nodejs && npm install  -global serverless@1.30.0 @angular/cli@1.7.3 jshint
+# Install scripts for golang 
+RUN curl -O https://storage.googleapis.com/golang/go${goVersion}.linux-amd64.tar.gz && tar -xvf go${goVersion}.linux-amd64.tar.gz && mv go /usr/local
+ENV GOROOT /usr/local/go 
+ENV GOBIN  /usr/local/go/bin 
+ENV GOPATH /usr/local/go/src 
+ENV PATH $GOROOT:$GOBIN:$GOPATH:$PATH 
+RUN curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+
 # Copying plugins list. Downlading and installing plugins from Jenkins PluginsManager
 COPY dockerfiles/jenkins-ce/plugins.txt /usr/share/jenkins/ref/plugins.txt
 RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt

installscripts/jazz-terraform-unix-noinstances/api-gateway.tf

@@ -30,7 +30,7 @@ resource "aws_cloudwatch_log_group" "API-Gateway-Execution-Logs_prod" {
 
 resource "aws_cloudwatch_log_subscription_filter" "logfilter-dev" {
   name            = "logfilter-dev"
-  role_arn        = "${aws_iam_role.lambda_role.arn}"
+  role_arn        = "${aws_iam_role.platform_role.arn}"
   log_group_name  = "${aws_cloudwatch_log_group.API-Gateway-Execution-Logs_dev.name}"
   filter_pattern  = ""
   destination_arn = "${aws_kinesis_stream.logs_stream_prod.arn}"
@@ -39,7 +39,7 @@ resource "aws_cloudwatch_log_subscription_filter" "logfilter-dev" {
 
 resource "aws_cloudwatch_log_subscription_filter" "logfilter-stg" {
   name            = "logfilter-stg"
-  role_arn        = "${aws_iam_role.lambda_role.arn}"
+  role_arn        = "${aws_iam_role.platform_role.arn}"
   log_group_name  = "${aws_cloudwatch_log_group.API-Gateway-Execution-Logs_stg.name}"
   filter_pattern  = ""
   destination_arn = "${aws_kinesis_stream.logs_stream_prod.arn}"
@@ -48,13 +48,13 @@ resource "aws_cloudwatch_log_subscription_filter" "logfilter-stg" {
 
 resource "aws_cloudwatch_log_subscription_filter" "logfilter-prod" {
   name            = "logfilter-prod"
-  role_arn        = "${aws_iam_role.lambda_role.arn}"
+  role_arn        = "${aws_iam_role.platform_role.arn}"
   log_group_name  = "${aws_cloudwatch_log_group.API-Gateway-Execution-Logs_prod.name}"
   filter_pattern  = ""
   destination_arn = "${aws_kinesis_stream.logs_stream_prod.arn}"
   distribution    = "Random"
 }
 
 resource "aws_api_gateway_account" "cloudwatchlogroleupdate" {
-  cloudwatch_role_arn = "${aws_iam_role.lambda_role.arn}"
+  cloudwatch_role_arn = "${aws_iam_role.platform_role.arn}"
 }

installscripts/jazz-terraform-unix-noinstances/chefProvisioner.tf

@@ -80,11 +80,7 @@ resource "null_resource" "configureJenkinsInstance" {
     inline = "mkdir -p ${var.chefDestDir}"
   }
 
-  #Copy the chef playbooks and jenkins binary plugin blobs over to the remote Jenkins server
-  provisioner "file" {
-    source      = "${var.jenkinsPluginsSourceDir}"
-    destination = "${var.chefDestDir}/"
-  }
+  #Copy the chef playbooks over to the remote Jenkins server
 
   provisioner "file" {
     source      = "${var.cookbooksSourceDir}"
@@ -115,7 +111,7 @@ resource "null_resource" "configureJenkinsDocker" {
 resource "null_resource" "postJenkinsConfiguration" {
   depends_on = ["null_resource.configureJenkinsInstance", "null_resource.configureJenkinsDocker", "aws_elasticsearch_domain.elasticsearch_domain"]
   provisioner "local-exec" {
-    command = "${var.modifyCodebase_cmd}  ${lookup(var.jenkinsservermap, "jenkins_security_group")} ${lookup(var.jenkinsservermap, "jenkins_subnet")} ${aws_iam_role.lambda_role.arn} ${var.region} ${var.envPrefix} ${var.cognito_pool_username}"
+    command = "${var.modifyCodebase_cmd}  ${lookup(var.jenkinsservermap, "jenkins_security_group")} ${lookup(var.jenkinsservermap, "jenkins_subnet")} ${aws_iam_role.platform_role.arn} ${var.region} ${var.envPrefix} ${var.cognito_pool_username}"
   }
 
   // Injecting bootstrap variables into Jazz-core Jenkinsfiles*

installscripts/jazz-terraform-unix-noinstances/iam.tf

@@ -1,10 +1,10 @@
 resource "aws_iam_role_policy_attachment" "lambdafullaccess" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AWSLambdaFullAccess"
 }
 
 resource "aws_iam_role_policy_attachment" "apigatewayinvokefullAccess" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonAPIGatewayInvokeFullAccess"
 }
 
@@ -13,33 +13,39 @@ resource "aws_iam_role_policy_attachment" "cloudwatchlogaccess" {
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
 }
 
+resource "aws_iam_role_policy_attachment" "cloudwatchlogaccessbasic" {
+  role       = "${aws_iam_role.platform_role.name}"
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
+}
+
 resource "aws_iam_role_policy_attachment" "kinesisaccess" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonKinesisFullAccess"
 }
 
 resource "aws_iam_role_policy_attachment" "s3fullaccess" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonS3FullAccess"
 }
 
 resource "aws_iam_role_policy_attachment" "sqsfullaccess" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonSQSFullAccess"
 }
 
 resource "aws_iam_role_policy_attachment" "cognitopoweruser" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonCognitoPowerUser"
 }
 
 resource "aws_iam_role_policy_attachment" "pushtocloudwatchlogs" {
-  role       = "${aws_iam_role.lambda_role.name}"
+  role       = "${aws_iam_role.platform_role.name}"
   policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonAPIGatewayPushToCloudWatchLogs"
 }
 
 resource "aws_iam_role" "lambda_role" {
   name = "${var.envPrefix}_basic_execution"
+  tags = "${merge(var.additional_tags, local.common_tags)}"
   assume_role_policy = <<EOF
 {
   "Version": "2012-10-17",
@@ -71,9 +77,9 @@ resource "aws_iam_role" "lambda_role" {
 EOF
 }
 
-resource "aws_iam_role_policy" "basic_execution_policy" {
-  name = "${var.envPrefix}_basic_execution_policy"
-  role = "${aws_iam_role.lambda_role.id}"
+resource "aws_iam_role_policy" "platform_service_policy" {
+  name = "${var.envPrefix}_platform_service_policy"
+  role = "${aws_iam_role.platform_role.id}"
 
   policy = <<EOF
 {
@@ -90,3 +96,37 @@ resource "aws_iam_role_policy" "basic_execution_policy" {
 }
 EOF
 }
+
+resource "aws_iam_role" "platform_role" {
+  name = "${var.envPrefix}_platform_services"
+  tags = "${merge(var.additional_tags, local.common_tags)}"
+  assume_role_policy = <<EOF
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+        "Sid": "",
+        "Effect": "Allow",
+        "Principal": {
+            "Service": "apigateway.amazonaws.com"
+        },
+        "Action": "sts:AssumeRole"
+    },
+    {
+        "Effect": "Allow",
+        "Principal": {
+            "Service": "lambda.amazonaws.com"
+        },
+        "Action": "sts:AssumeRole"
+    },
+    {
+        "Effect": "Allow",
+        "Principal": {
+            "Service": "logs.${var.region}.amazonaws.com"
+        },
+        "Action": "sts:AssumeRole"
+    }
+  ]
+}
+EOF
+}

installscripts/jazz-terraform-unix-noinstances/jenkins.tf

@@ -46,7 +46,10 @@ resource "null_resource" "update_jenkins_configs" {
     command = "${var.configureS3Names_cmd} ${aws_s3_bucket.oab-apis-deployment-dev.bucket} ${aws_s3_bucket.oab-apis-deployment-stg.bucket} ${aws_s3_bucket.oab-apis-deployment-prod.bucket} ${aws_s3_bucket.jazz-web.bucket} ${var.jenkinsjsonpropsfile}"
   }
   provisioner "local-exec" {
-    command = "${var.modifyPropertyFile_cmd} ROLEID ${aws_iam_role.lambda_role.arn}  ${var.jenkinsjsonpropsfile}"
+    command = "${var.modifyPropertyFile_cmd} PLATFORMSERVICES_ROLEID ${aws_iam_role.platform_role.arn}  ${var.jenkinsjsonpropsfile}"
+  }
+  provisioner "local-exec" {
+    command = "${var.modifyPropertyFile_cmd} USERSERVICES_ROLEID ${aws_iam_role.lambda_role.arn}  ${var.jenkinsjsonpropsfile}"
   }
   provisioner "local-exec" {
     command = "${var.modifyPropertyFile_cmd} WEBSITE_DEV_BUCKET ${aws_s3_bucket.dev-serverless-static.bucket} ${var.jenkinsjsonpropsfile}"

installscripts/jazz-terraform-unix-noinstances/s3bucket.tf

@@ -157,7 +157,7 @@ data "aws_iam_policy_document" "dev-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.dev-serverless-static.arn}/*"
@@ -170,7 +170,7 @@ data "aws_iam_policy_document" "dev-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.dev-serverless-static.arn}"
@@ -191,7 +191,7 @@ data "aws_iam_policy_document" "stg-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.stg-serverless-static.arn}/*"
@@ -204,7 +204,7 @@ data "aws_iam_policy_document" "stg-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.stg-serverless-static.arn}"
@@ -227,7 +227,7 @@ data "aws_iam_policy_document" "prod-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.prod-serverless-static.arn}/*"
@@ -240,7 +240,7 @@ data "aws_iam_policy_document" "prod-serverless-static-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.prod-serverless-static.arn}"
@@ -261,7 +261,7 @@ data "aws_iam_policy_document" "jazz-web-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.jazz-web.arn}/*"
@@ -275,7 +275,7 @@ data "aws_iam_policy_document" "jazz-web-policy-data-contents" {
     ]
     principals  {
       type="AWS",
-      identifiers = ["${aws_iam_role.lambda_role.arn}"]
+      identifiers = ["${aws_iam_role.platform_role.arn}"]
     }
     resources = [
       "${aws_s3_bucket.jazz-web.arn}"