Merge pull request #325 from tmobile/v1.8.1

Release v1.9

Former-commit-id: 2f4a896

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 120
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist

.gitignore

@@ -12,4 +12,5 @@ node_modules
 */target/
 .nyc_output/
 npm-debug.log
-*/package-lock.json
+*/package-lock.json
+.terraform/

Installer.sh

@@ -20,6 +20,7 @@ LOG_FILE_NAME=installer_setup.out
 LOG_FILE=`realpath $INSTALL_DIR/$LOG_FILE_NAME`
 JAZZ_BRANCH="master"
 CODE_QUALITY='false'
+AWS_TAGS=''
 
 function start_wizard {
     # Set the permissions
@@ -29,7 +30,7 @@ function start_wizard {
     # Call the python script to continue installation process
     cd $INSTALL_DIR/installscripts/wizard
 
-    python ./run.py $JAZZ_BRANCH $INSTALL_DIR $CODE_QUALITY
+    python ./run.py $JAZZ_BRANCH $INSTALL_DIR $CODE_QUALITY "$AWS_TAGS"
 
     setterm -term linux -fore green
     setterm -term linux -fore default
@@ -76,11 +77,11 @@ while [ $# -gt 0 ] ; do
             shift
             while [ "$#" -gt 0 ] ; do
                 if [[ "$1" =~ Key=.*,Value=.* ]] && [[ ! "$1" =~ ";" ]] ; then
-                    arr+=("$1")
+                    AWS_TAGS="$AWS_TAGS $1"
                 elif [[ $1 == -* ]] ; then break
                 else
-                    echo "Please specify tags in format: Key=stackName,Value=production"
-                    echo "Usage: ./Installer --tags Key=stackName,Value=production Key=department,Value=devops"
+                    echo "Please specify tags in format: Key=stackname,Value=production"
+                    echo "Usage: ./Installer --tags Key=stackname,Value=production Key=department,Value=devops"
                     exit 1
                 fi
                 shift

README.md

@@ -25,6 +25,14 @@ You can try out public preview version of Jazz by registering with your email ad
 
 For more details, see the [Wiki](https://github.com/tmobile/jazz-installer/wiki).
 
+## Development
+
+If you're interested in submitting a PR, it would be a good idea to set up your editor/IDE to use the following checkers:
+editorconfig
+flake8 for Python linting
+tflint for Terraform script linting
+foodcritic for Chef script linting
+
 ## License
 
 Jazz is released under the [Apache 2.0 License](http://www.apache.org/licenses/LICENSE-2.0).

installscripts/cookbooks/Policyfile.rb

@@ -4,4 +4,4 @@
 cookbook 'nodejs', '~> 5.0.0'
 cookbook 'cloudcli', '~> 1.2.0'
 cookbook 'jenkins', path: './jenkins'
-run_list 'jenkins::prereqs', 'jenkins::sonarqube-scanner', 'jenkins::configurejenkins'
+run_list 'jenkins::setupjenkins', 'jenkins::sonarqube-scanner', 'jenkins::configurejenkins'

installscripts/cookbooks/jenkins/attributes/default.rb

@@ -25,6 +25,7 @@
 default['bbpassword'] = 'REPLACEME'
 default['sonaruser'] = 'REPLACEME'
 default['sonarpassword'] = 'REPLACEME'
+default['dockerizedJenkins'] = false
 
 #Maven cookbook property
 default['maven']['version'] = '3.5.2'

installscripts/cookbooks/jenkins/files/default/jazz-installer-vars.json

@@ -32,7 +32,8 @@
             "DEV_BUCKET": "",
             "STG_BUCKET": "",
             "PROD_BUCKET": ""
-        }
+        },
+        "TAGS": []
     },
     "REPOSITORY": {
         "BASE_URL": "",
@@ -108,6 +109,8 @@
     "SLACK": {
         "ENABLE_SLACK": "false",
         "SLACK_USER": "{slack_user}",
-        "SLACK_TOKEN": "{slack_token}"
+        "SLACK_TOKEN": "{slack_token}",
+        "SLACK_WORKSPACE": "{slack_workspace}",
+        "SLACK_SVC_ID": "{slack_svc_acc_id}"
     }
 }

installscripts/cookbooks/jenkins/files/jobs/job_jazz_ui.sh

@@ -42,7 +42,7 @@ cat <<EOF | $JENKINS_CLI_CMD create-job "jazz_ui"
       <configVersion>2</configVersion>
       <userRemoteConfigs>
         <hudson.plugins.git.UserRemoteConfig>
-          <url>http://$SCM_ELB/slf/jazz-ui.git</url>
+          <url>http://$SCM_ELB/slf/jazz_ui.git</url>
           <credentialsId>jazz_repocreds</credentialsId>
         </hudson.plugins.git.UserRemoteConfig>
       </userRemoteConfigs>

installscripts/cookbooks/jenkins/recipes/configurejenkins.rb

@@ -1,25 +1,13 @@
-# TODO: These plugins should be `curl`-ed from a server here,
-# they should not be part of this cookbook.
-execute 'concatJenkinsPlugins' do
-  command "cat #{node['chef_root']}/jenkinsplugins/plugins0* > #{node['chef_root']}/plugins.tar"
-end
-
-execute 'extractJenkinsPlugins' do
-  command "tar -xf #{node['chef_root']}/plugins.tar -C /var/lib/jenkins/"
-end
-
-# Clean up the plugin tar from previous step, it is rather large
-file "#{node['chef_root']}/plugins.tar" do
-  action :delete
-end
-
-# Copy authfile
-cookbook_file "#{node['chef_root']}/authfile" do
-  source 'authfile'
-  action :create
+#
+if node['dockerizedJenkins'] == false
+  # Copy authfile
+  cookbook_file "#{node['chef_root']}/authfile" do
+    source 'authfile'
+    action :create
+  end
 end
 
-directory '/var/lib/jenkins/workspace' do
+directory "#{node['jenkins']['home']}/workspace" do
   owner 'jenkins'
   group 'jenkins'
   mode '0777'
@@ -37,14 +25,6 @@
   command "sudo sed -i 's/JENKINS_JAVA_OPTIONS=.*.$/JENKINS_JAVA_OPTIONS=\"-Djava.awt.headless=true -Xmx1024m -XX:MaxPermSize=512m\"/' /etc/sysconfig/jenkins"
 end
 
-service 'jenkins' do
-  action :restart
-end
-
-# Wait a bit, Java apps don't coldboot very quickly...
-execute 'waitForFirstJenkinsRestart' do
-  command 'sleep 30'
-end
 
 # Try to fetch the version-appropriate Jenkins CLI jar from the server itself.
 execute 'copyJenkinsClientJar' do
@@ -66,30 +46,30 @@
   source 'xmls.tar'
   action :create
 end
-
+#ToDo ChefRemoval
 execute 'extractXmls' do
   command "tar -xvf #{node['chef_root']}/xmls.tar"
-  cwd '/var/lib/jenkins'
+  cwd "#{node['jenkins']['home']}"
 end
 
-cookbook_file '/var/lib/jenkins/config.xml' do
+cookbook_file "#{node['jenkins']['home']}/config.xml" do
   source 'config.xml'
   action :create
 end
 
-cookbook_file '/var/lib/jenkins/scriptApproval.xml' do
+cookbook_file "#{node['jenkins']['home']}/scriptApproval.xml" do
   source 'scriptApproval.xml'
   action :create
 end
 
-cookbook_file '/var/lib/jenkins/credentials.xml' do
+cookbook_file "#{node['jenkins']['home']}/credentials.xml" do
   source 'credentials.xml'
   action :create
 end
 
 bash 'configJenkinsLocConfigXml' do
   code <<-SCRIPT
-  JENKINS_LOC_CONFIG_XML=/var/lib/jenkins/jenkins.model.JenkinsLocationConfiguration.xml
+  JENKINS_LOC_CONFIG_XML=#{node['jenkins']['home']}/jenkins.model.JenkinsLocationConfiguration.xml
 
   sed  -i "s=adminAddress.*.$=adminAddress>#{node['jenkins']['SES-defaultSuffix']}</adminAddress>=g" $JENKINS_LOC_CONFIG_XML
   sed  -i "s=jenkinsUrl.*.$=jenkinsUrl>http://#{node['jenkinselb']}/</jenkinsUrl>=g" $JENKINS_LOC_CONFIG_XML
@@ -102,7 +82,7 @@
   # Configure Gitlab Plugin
   bash 'configuregitlabplugin' do
     code <<-SCRIPT
-    sed -i "s/ip/#{node['scmelb']}/g" /var/lib/jenkins/com.dabsquared.gitlabjenkins.connection.GitLabConnectionConfig.xml
+    sed -i "s/ip/#{node['scmelb']}/g" #{node['jenkins']['home']}/com.dabsquared.gitlabjenkins.connection.GitLabConnectionConfig.xml
   SCRIPT
   end
 
@@ -141,16 +121,6 @@
   end
 end
 
-# TODO: we do this at the end, do we need it here?
-service 'jenkins' do
-  supports [:stop, :start, :restart]
-  action [:restart]
-end
-
-# Wait a bit, Java apps don't coldboot very quickly...
-execute 'waitForSecondJenkinsRestart' do
-  command 'sleep 30'
-end
 
 # If this happens to already exist, remove it before we clone again.
 directory "#{node['chef_root']}/jazz-core" do
@@ -275,13 +245,15 @@
   command "#{node['script_root']}/job_jazz_ui.sh #{node['jenkins']['clicommand']} #{node['scmpath']}"
 end
 
-directory '/var/lib/jenkins' do
+directory "#{node['jenkins']['home']}" do
   owner 'jenkins'
   group 'jenkins'
   action :create
 end
-
-service 'jenkins' do
-  supports [:stop, :start, :restart]
-  action [:restart]
+#Docker container has no service jenkins
+if node['dockerizedJenkins'] == false
+  service 'jenkins' do
+    supports [:stop, :start, :restart]
+    action [:restart]
+  end
 end

installscripts/cookbooks/jenkins/recipes/setupjenkins.rb

@@ -0,0 +1,42 @@
+if node['dockerizedJenkins'] == false
+  #prerequisites
+  apt_update 'update' #force-update apt cache on Debian-derivatives to avoid pkg fetch errors
+  package 'git'
+  include_recipe 'maven::default'
+  include_recipe 'nodejs'
+  include_recipe 'cloudcli'
+
+  npm_package 'serverless' do
+    version '1.26.1'
+  end
+
+  npm_package '@angular/cli' do
+    version '1.7.3'
+  end
+
+  npm_package 'jshint'
+
+  #plugin management
+  execute 'concatJenkinsPlugins' do
+    command "cat #{node['chef_root']}/jenkinsplugins/plugins0* > #{node['chef_root']}/plugins.tar"
+  end
+
+  execute 'extractJenkinsPlugins' do
+    command "tar -xf #{node['chef_root']}/plugins.tar -C #{node['jenkins']['home']}/"
+  end
+
+  # Clean up the plugin tar from previous step, it is rather large
+  file "#{node['chef_root']}/plugins.tar" do
+    action :delete
+  end
+
+  service 'jenkins' do
+    action :restart
+  end
+
+  # Wait a bit, Java apps don't coldboot very quickly...
+  execute 'waitForFirstJenkinsRestart' do
+    command 'sleep 30'
+  end
+
+end

installscripts/dockerfiles/jenkins-ce/Dockerfile

@@ -0,0 +1,31 @@
+# Base Image and maintainer info
+FROM jenkins/jenkins:lts
+MAINTAINER JazzOSS Team
+
+# Switching to root to configure the image with system packages
+USER root
+RUN apt-get update && apt-get install -y vim curl sudo libtool autoconf make unzip rsync gcc autogen shtool pkg-config lsb-release python python-dev python-pip python-setuptools groff less && \
+curl https://bootstrap.pypa.io/get-pip.py | python && pip install --upgrade awscli && apt-get clean && pip install virtualenv && /usr/bin/easy_install virtualenv
+RUN wget -O /opt/apache-maven-3.5.2-bin.tar.gz https://archive.apache.org/dist/maven/maven-3/3.5.2/binaries/apache-maven-3.5.2-bin.tar.gz && tar xzvf /opt/apache-maven-3.5.2-bin.tar.gz -C  /opt && export PATH=$PATH:/opt/apache-maven-3.5.2/bin >> /etc/profile.d/maven.sh && ln -sf /opt/apache-maven-3.5.2/bin/mvn /usr/bin/mvn
+RUN curl -sL https://deb.nodesource.com/setup_8.x | bash && apt-get install -y nodejs && npm install  -global serverless@1.26.1 @angular/cli@1.7.3 jshint
+# Copying plugins list. Downlading and installing plugins from Jenkins PluginsManager
+COPY dockerfiles/jenkins-ce/plugins.txt /usr/share/jenkins/ref/plugins.txt
+RUN /usr/local/bin/install-plugins.sh < /usr/share/jenkins/ref/plugins.txt
+
+# Copy the chef recipes, chefconfigs and bootstrap script
+COPY --chown=jenkins:jenkins cookbooks  /tmp/jazz-chef/cookbooks
+COPY --chown=jenkins:jenkins dockerfiles/jenkins-ce/chefInstaller.sh /tmp/jazz-chef/chefInstaller.sh
+
+RUN chown -R jenkins:jenkins /tmp/jazz-chef
+# Creating node_path directory
+RUN mkdir /var/jenkins_home/node
+
+# Setting execute bit and executing the bootstrap script
+RUN chmod 777 /tmp/jazz-chef/chefInstaller.sh
+RUN bash /tmp/jazz-chef/chefInstaller.sh
+
+# Switching back to Jenkins user for safety
+USER jenkins
+
+# Expose the jenkins port
+EXPOSE 8081

installscripts/dockerfiles/jenkins-ce/chefInstaller.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Installing chefclient
+curl -o chefdk.deb https://packages.chef.io/files/stable/chefdk/3.0.36/ubuntu/16.04/chefdk_3.0.36-1_amd64.deb
+dpkg -i chefdk.deb
+
+# Installing pip
+# curl -sL -O https://bootstrap.pypa.io/get-pip.py && python get-pip.py
+# chmod -R o+w /usr/lib/python2.7/* /usr/bin/

installscripts/dockerfiles/jenkins-ce/chefclient.sh

@@ -0,0 +1,46 @@
+#Spin wheel for visual effects
+spin_wheel()
+{
+    RED='\033[0;31m'
+    GREEN='\033[0;32m'
+    NC='\033[0m'
+
+    pid=$1 # Process Id of the previous running command
+    message=$2
+    spin='-\|/'
+    printf "\r$message...."
+    i=0
+
+    while ps -p $pid > /dev/null
+    do
+        #echo $pid $i
+        i=$(( (i+1) %4 ))
+        printf "\r${GREEN}$message....${spin:$i:1}"
+        sleep .05
+    done
+
+    wait "$pid"
+    exitcode=$?
+    if [ $exitcode -gt 0 ]
+    then
+        printf "\r${RED}$message....Failed${NC}\n"
+        exit
+    else
+        printf "\r${GREEN}$message....Completed${NC}\n"
+
+    fi
+}
+
+
+cd ~/jazz-installer/installscripts
+sudo docker cp cookbooks/. jenkins-server:/tmp/jazz-chef/cookbooks/
+sudo docker cp jenkinsplugins jenkins-server:/tmp/jazz-chef/
+# Running chef-client to execute cookbooks
+sudo docker exec -u root -i jenkins-server bash -c "chef install /tmp/jazz-chef/cookbooks/Policyfile.rb && chef export /tmp/jazz-chef/cookbooks/Policyfile.rb /tmp/jazz-chef/chef-export && cd /tmp/jazz-chef/chef-export && sudo chef-client -z"
+
+# Once the docker image is configured, we will commit the image.
+sudo docker commit -m "JazzOSS-Custom Jenkins container" jenkins-server jazzoss-jenkins-server
+sudo docker restart jenkins-server
+# The image jazzoss-jenkins-server is now ready to be shipped to and/or spinned in any docker hosts like ECS cluster/fargate etc.
+sleep 20 &
+spin_wheel $! "Initializing the Jenkins container"