[CE-95] Added ansible fabric deploy to cello

fabric deploy module is a ansible project which automates fabric
provisioning, build, deploy onto various cloud and bare metal
environment. Please see the README.md file at the root directory
of the module for details.

Change-Id: I52a4378e9ab2c6d7b9d079aa09f3d0d8b36e7db7
Signed-off-by: tongliofcary <litong01@us.ibm.com>

.gitignore

@@ -217,3 +217,8 @@ jspm_packages
 .npm
 .eslintcache
 !/**/lib
+
+# Ansible modules
+src/agent/ansible/run/*
+src/agent/ansible/vars/*
+src/agent/ansible/*.retry

src/agent/ansible/README.md

@@ -0,0 +1,67 @@
+# Hyperledger Fabric Deployment
+
+Hyperledger fabric deploy module is an ansible project which deploys hybperledger fabric
+onto multiple physical or virtual servers.
+
+ - Provision virtual servers to participate in fabric network
+ - Install necessary hyperledger dependent libraries and packages
+ - Setup kubernetes 1.7.0 or overlay network so that containers can communicate cross multiple docker hosts
+ - Install registrator and dns services so that containers can be referenced by name
+ - Build hyperledger fabric artifacts (optional)
+ - Run hyperledger fabric tests (optional)
+ - Generate fabric network certificats, genesis block, transaction blocks
+ - Push new or tagged fabric images onto all docker hosts
+ - Deploy fabric network
+ - Join peers to channels, instantiate chaincode
+
+## Status
+
+In development
+
+## Requirements for ansible controller
+
+- [Install Ansible](http://docs.ansible.com/ansible/intro_installation.html)
+- [Ubuntu 16.04 machines] (https://cloud-images.ubuntu.com/releases/16.04/)
+- Install cloud platform dependent packages such as OpenStack shade or AWS boto
+- Ansible 2.3.0.0 or above
+
+Here is an example on how to make a clean ubuntu system as your ansible controller
+If you have other system as your Ansible controller, you can do similar steps to setup
+the environment, the command may not be exact the same but the steps you
+need to do should be identical.
+
+    sudo apt-get update
+    sudo apt-get install python-dev python-pip libssl-dev libffi-dev -y
+    sudo pip install --upgrade pip
+    sudo pip install six==1.10.0
+    sudo pip install ansible==2.3.0.0
+    git clone https://github.com/litong01/fabric-deploy.git
+
+## Deploy hyperledger fabric onto different environment
+
+### On VirtualBox::
+
+    1. make changes to vars/vb.yml according to your VirtualBox environment
+    2. export password="your password to vb env"
+    3. To stand up the fabric network::
+        ansible-playbook -e "mode=apply" vb.yml
+    4. To tear down the fabric network::
+        ansible-playbook -e "mode=destroy" vb.yml
+
+### On OpenStack cloud::
+
+    1. make changes to vars/os.yml according to your OpenStack cloud
+    2. export password="your password of your OpenStack cloud account"
+    3. To stand up the fabric network::
+        ansible-playbook -e "mode=apply" os.yml
+    4. To tear down the fabric network::
+        ansible-playbook -e "mode=destroy" os.yml
+
+### On AWS cloud::
+
+    1. make changes to vars/aws.yml according to your aws cloud
+    2. export AWS_SECRET_KEY="your secret key of your aws account"
+    3. To stand up the fabric network::
+        ansible-playbook -e "mode=apply" aws.yml
+    4. To tear down the fabric network::
+        ansible-playbook -e "mode=destroy" aws.yml

src/agent/ansible/ansible.cfg

@@ -0,0 +1,4 @@
+[defaults]
+log_path=./run/ansible.log
+inventory = ./hosts
+host_key_checking = False

src/agent/ansible/aws.yml

@@ -0,0 +1,34 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  vars:
+    env: "aws"
+    cloud_type: "aws"
+  include: "roles/cloud_aws/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "aws"
+    env_type: "flanneld"
+  include: "roles/env_flanneld/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "bc1st"
+    deploy_type: "compose"
+  include: "roles/deploy_compose/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/awsk8s.yml

@@ -0,0 +1,34 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  vars:
+    env: "aws"
+    cloud_type: "aws"
+  include: "roles/cloud_aws/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "aws"
+    env_type: "k8s"
+  include: "roles/env_k8s/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "bc1st"
+    deploy_type: "k8s"
+  include: "roles/deploy_k8s/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/azureyml

@@ -0,0 +1,34 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  vars:
+    env: "azure"
+    cloud_type: "azure"
+  include: "roles/cloud_azure/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "azure"
+    env_type: "k8s"
+  include: "roles/env_k8s/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "bc1st"
+    deploy_type: "k8s"
+  include: "roles/deploy_k8s/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/group_vars/all.yml

@@ -0,0 +1,7 @@
+---
+etcdtoken: "{{ lookup('password',
+             '/tmp/etcdtoken chars=ascii_letters,digits length=16') }}"
+
+proxy_env: {
+}
+

src/agent/ansible/hosts

@@ -0,0 +1 @@
+cloud ansible_host=127.0.0.1 ansible_python_interpreter=python

src/agent/ansible/initcluster.yml

@@ -0,0 +1,19 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  include: "roles/env_{{ env_type | default('flanneld') }}/plays.yml"
+
+- name: inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/os.yml

@@ -0,0 +1,34 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  vars:
+    env: "os"
+    cloud_type: "os"
+  include: "roles/cloud_os/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "os"
+    env_type: "flanneld"
+  include: "roles/env_flanneld/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "bc1st"
+    deploy_type: "compose"
+  include: "roles/deploy_compose/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/osk8s.yml

@@ -0,0 +1,34 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  vars:
+    env: "os"
+    cloud_type: "os"
+  include: "roles/cloud_os/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "os"
+    env_type: "k8s"
+  include: "roles/env_k8s/plays.yml"
+
+- name: Run the plays
+  vars:
+    env: "bc1st"
+    deploy_type: "k8s"
+  include: "roles/deploy_k8s/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/provcluster.yml

@@ -0,0 +1,19 @@
+---
+- name: Get start timestamp
+  hosts: cloud
+  connection: local
+  tasks:
+    - set_fact:
+        starttime: "{{ ansible_date_time }}"
+
+- name: Run the plays
+  include: "roles/cloud_{{ cloud_type | default('os') }}/plays.yml"
+
+- name: Inform the installer
+  hosts: cloud
+  connection: local
+  tasks:
+    - debug:
+        msg: >-
+          The work load started at {{ hostvars.cloud.starttime.time }},
+          ended at {{ ansible_date_time.time }}

src/agent/ansible/roles/cloud_aws/README.md

@@ -0,0 +1,19 @@
+Provision VMs against AWS cloud
+
+Ansible uses Boto for AWS interactions, so you'll need that installed on your control host.
+We're also going to make some use of the AWS CLI tools, so get those too. Your platform may
+differ, but the following will work for most platforms:
+
+https://atplanet.co/blog/ec2-auto-scaling-with-ansible.html
+--------
+Pre-reqs
+--------
+
+To use ansible to work with AWS, python library like boto and boto3 must be installed:
+---------
+sudo pip install boto boto3
+
+Nice to have:
+-------------
+sudo apt-get install awscli -y
+sudo apt-get install ec2-api-tools -y

src/agent/ansible/roles/cloud_aws/plays.yml

@@ -0,0 +1,40 @@
+---
+- name: Prepare to run the workload
+  hosts: cloud
+  connection: local
+  vars_files:
+    - "./../../vars/{{ env }}.yml"
+  tasks:
+    - include: "prepare/tasks/{{ mode }}.yml"
+  tags: "prepare"
+
+- name: provision servers
+  hosts: prohosts
+  connection: local
+  strategy: free
+  vars_files:
+    - "{{ playbook_dir }}/../../vars/{{ env }}.yml"
+  tasks:
+    - include: "provision/tasks/{{ mode }}.yml"
+  tags: "provision"
+
+- name: Post provision process
+  hosts: cloud
+  connection: local
+  vars_files:
+    - "{{ playbook_dir }}/../../vars/{{ env }}.yml"
+  tasks:
+    - include: "postprovision/tasks/{{ mode }}.yml"
+  tags: "postprovision"
+
+- name: change /etc/hosts files so that each node knows other nodes
+  hosts: allnodes
+  strategy: free
+  user: "{{ cluster.ssh_user }}"
+  become: true
+  become_user: root
+  vars_files:
+    - "{{ playbook_dir }}/../../vars/{{ env }}.yml"
+  tasks:
+    - include: "../cloud_os/namenode/tasks/{{ mode }}.yml"
+  tags: "namenode"

src/agent/ansible/roles/cloud_aws/postprovision/tasks/apply.yml

@@ -0,0 +1,2 @@
+---
+- include: ../../../cloud_os/postprovision/tasks/apply.yml

src/agent/ansible/roles/cloud_aws/postprovision/tasks/destroy.yml

@@ -0,0 +1,22 @@
+---
+- name: Setup env specific variables
+  set_fact:
+    aws_usr_key: "{{ auth.username }}"
+    aws_sec_key: "{{ auth.password }}"
+    region: "{{ cluster.region_name | default('us-east-1') }}"
+
+- name: Delete aws key pairs
+  ec2_key:
+    name: "{{ cluster.ssh_key_name }}"
+    aws_access_key: "{{ aws_usr_key }}"
+    aws_secret_key: "{{ aws_sec_key }}"
+    region: "{{ region }}"
+    state: absent
+
+- name: Delete EC2 security group
+  ec2_group:
+    name: "{{ cluster.security_group }}"
+    aws_access_key: "{{ aws_usr_key }}"
+    aws_secret_key: "{{ aws_sec_key }}"
+    region: "{{ region }}"
+    state: absent