From 29a108eef652007d30709bb9a4c2ada074cee1e2 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:51:36 -0400 Subject: [PATCH 01/16] Added provide_default filter to provide default values if value is null string Current built in default jinja filter helps the user to omit/provide default to keys in the ansible module. However , it doesnt provide a way for user to conditionally provide a default value to ie., based on the value provided by the output of previous filter . In order to facilitate the same provide_default filter is being written. --- .../provision/filter_plugins/provide_default.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 linchpin/provision/filter_plugins/provide_default.py diff --git a/linchpin/provision/filter_plugins/provide_default.py b/linchpin/provision/filter_plugins/provide_default.py new file mode 100755 index 000000000..8f615684e --- /dev/null +++ b/linchpin/provision/filter_plugins/provide_default.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +import os + +def provide_default(fetched, default): + if fetched == "": + return default + +class FilterModule(object): + ''' A filter to fix interface's name format ''' + def filters(self): + return { + 'provide_default': provide_default + } From c3f6f4eda640ef29d9d98ee3f1cf3ee068d666da Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:51:56 -0400 Subject: [PATCH 02/16] added auth_driver module to fetch credentials from specified locations auth_driver module fetches and parses file returns to credentials in auth_var variable making creds accessible inside ansible playbooks. --- linchpin/provision/library/auth_driver.py | 86 +++++++++++++++++++---- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/linchpin/provision/library/auth_driver.py b/linchpin/provision/library/auth_driver.py index 3f004865f..e240f3b31 100644 --- a/linchpin/provision/library/auth_driver.py +++ b/linchpin/provision/library/auth_driver.py @@ -13,11 +13,24 @@ - This module allows a user to fetch credentials on request and egister it as variable in ansible. options: - type: + name: description: - type of credential required + name of the credential file to be used required: true - + cred_type: + description: + credential type , type of credential to be used. + eg: aws, gcloud , openstack , etc., + required: false + cred_path: + description: + credentials path where the credentials are to be stored + required: false + driver: + description: + defaults to file type. + required: true + author: Samvaran Kashyap Rallabandi - ''' @@ -29,28 +42,71 @@ import shlex import tempfile import yaml +import glob +try: + import configparser as ConfigParser +except ImportError: + import ConfigParser as ConfigParser -def check_file_paths(module, *args): - for file_path in args: - if not os.path.exists(file_path): - module.fail_json(msg= "File not found %s not found" % (file_path)) - if not os.access(file_path, os.R_OK): - module.fail_json(msg= "File not accesible %s not found" % (file_path)) - if os.path.isdir(file_path): - module.fail_json(msg= "Recursive directory not supported %s " % (file_path)) +class ConfigDict(ConfigParser.ConfigParser): + def as_dict(self): + d = dict(self._sections) + for k in d: + d[k] = dict(self._defaults, **d[k]) + d[k].pop('__name__', None) + return d + +def list_files(path): + return glob.glob(path+"/*.*") + +def parse_file(filename): + cred_str = open(filename, "r").read() + try: + out = json.loads(cred_str) + except Exception as e: + try: + out = yaml.load(cred_str) + except Exception as e: + try: + config = ConfigDict() + f = open(filename) + config.readfp(f) + out = config.as_dict() + f.close() + except Exception as e: + module.fail_json(msg= "Error {0} ".format(str(e))) + return out + +def get_cred(name, creds_path): + paths = creds_path.split(";") + files = [] + for path in paths: + files = list_files(path) + for filename in files: + if name == filename.split("/")[-1].split(".")[0]: + out = parse_file(filename) + return out, path + module.fail_json(msg= "Error: Credential not found") def main(): + global module module = AnsibleModule( argument_spec={ - 'type': {'required': True, 'aliases': ['auth_type']}, - 'creds_store': {'required': False, 'aliases': ['credential_store']}, + 'name': {'required': True, 'aliases': ['name']}, + 'cred_type': {'required': False, 'aliases': ['credential_type']}, + 'cred_path': {'required': True, 'aliases': ['credential_store']}, + 'driver': {'required': True, 'aliases': ['driver_type']}, }, required_one_of=[], supports_check_mode=True ) + name = module.params["name"] + cred_type = module.params["cred_type"] + cred_path = module.params["cred_path"] + driver_type = module.params["driver"] + output, path = get_cred(name, cred_path) changed = True - module.exit_json(changed=changed, output={}) + module.exit_json(changed=changed, output=output, params=module.params, path=path) -from ansible.module_utils.basic import * main() From 594627c331a0460940385711ed9c6d738f432b58 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:52:18 -0400 Subject: [PATCH 03/16] updated schema v3 and schema_v4 to incorporate auth_driver functionality --- linchpin/defaults/schemas/schema_v3.json | 15 +++++++++++++-- linchpin/defaults/schemas/schema_v4.json | 13 ++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/linchpin/defaults/schemas/schema_v3.json b/linchpin/defaults/schemas/schema_v3.json index b1124b5db..116da4560 100644 --- a/linchpin/defaults/schemas/schema_v3.json +++ b/linchpin/defaults/schemas/schema_v3.json @@ -643,9 +643,20 @@ "assoc_creds": { "description":"contains creds file associated to this resource", "type":"string" - } + }, + "credentials": { + "type": "object", + "properties":{ + "name": { + "type": "string" + }, + "auth_type": { + "type": "string" + } + } + } }, - "required":["resource_group_name","res_group_type","res_defs","assoc_creds"], + "required":["resource_group_name","res_group_type","res_defs"], "additionalProperties": true }, "aws": { diff --git a/linchpin/defaults/schemas/schema_v4.json b/linchpin/defaults/schemas/schema_v4.json index 8896ae16b..d07b2e40e 100644 --- a/linchpin/defaults/schemas/schema_v4.json +++ b/linchpin/defaults/schemas/schema_v4.json @@ -705,7 +705,18 @@ }, "credentials": { "description":"contains creds file associated to this resource", - "type":"string" + "type":"object", + "properties": { + "profile": { + "type": "string" + }, + "auth_type": { + "type": "string" + }, + "name": { + "type": "string" + } + } } }, "required":["resource_group_name","resource_group_type","resource_definitions","credentials"], From a9341ad4f388cf2bc9d03b8255456fb185b41567 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:53:46 -0400 Subject: [PATCH 04/16] Added credentials folder to linchpin conf and templates --- linchpin/templates/credentials/.empty | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 linchpin/templates/credentials/.empty diff --git a/linchpin/templates/credentials/.empty b/linchpin/templates/credentials/.empty new file mode 100644 index 000000000..e69de29bb From 442e9e7c9ed582755b04183b2ac84e4ec1d18468 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:54:11 -0400 Subject: [PATCH 05/16] updated openstack playbooks to incorporate auth_driver functionality --- .../openstack/tasks/provision_os_heat.yml | 16 ++++---- .../openstack/tasks/provision_os_keypair.yml | 12 +----- .../openstack/tasks/provision_os_object.yml | 40 +++++-------------- .../openstack/tasks/provision_os_server.yml | 12 +----- .../roles/openstack/tasks/provision_os_sg.yml | 31 +++++--------- .../openstack/tasks/provision_os_volume.yml | 40 +++++-------------- .../tasks/provision_resource_group.yml | 24 +++++++++-- 7 files changed, 65 insertions(+), 110 deletions(-) diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_heat.yml b/linchpin/provision/roles/openstack/tasks/provision_os_heat.yml index 17d9dd29d..c3b6d03c9 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_heat.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_heat.yml @@ -16,10 +16,10 @@ template: "{{ res_def['template_path'] }}" state: "{{ state }}" wait: "yes" - os_username: "{{ username }}" - os_password: "{{ password }}" - os_tenant_name: "{{ project }}" - os_auth_url: "{{ endpoint }}" + os_auth_url: "{{ auth_var['auth_url']| default(omit) }}" + os_username: "{{ auth_var['username']| default(omit) }}" + os_password: "{{ auth_var['password']| default(omit) }}" + os_tenant_name: "{{ auth_var['project_name']| default(omit) }}" parameters: "{{ heat_params }}" register: res_def_output when: not async @@ -35,10 +35,10 @@ template: "{{ res_def['template_path'] }}" state: "{{ state }}" wait: "yes" - os_username: "{{ username }}" - os_password: "{{ password }}" - os_tenant_name: "{{ project }}" - os_auth_url: "{{ endpoint }}" + os_auth_url: "{{ auth_var['auth_url'] | default(omit) }}" + os_username: "{{ auth_var['username']| default(omit) }}" + os_password: "{{ auth_var['password']| default(omit) }}" + os_tenant_name: "{{ auth_var['project_name'] | default(omit) }}" parameters: "{{ heat_params }}" async: "{{ async_timeout }}" poll: 0 diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml b/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml index 14e4c1faa..757d748d2 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml @@ -1,10 +1,6 @@ - name: "provisioning/deprovisioning keypair" os_keypair: - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" @@ -24,11 +20,7 @@ - name: "Async:: provisioning/deprovisioning keypair" os_keypair: - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_object.yml b/linchpin/provision/roles/openstack/tasks/provision_os_object.yml index 5dd2a7685..00da759cd 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_object.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_object.yml @@ -5,11 +5,7 @@ - name: "provision/deprovision swift containers" os_object: - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" container: "{{ res_def['res_name'] | default(res_def['name']) }}" container_access: "{{ res_def['access'] | default('private') }}" wait: yes @@ -18,20 +14,13 @@ - name: "provision/deprovision swift containers when count specified" os_object: - auth: - auth_url: "{{ container.0 }}" - username: "{{ container.1 }}" - password: "{{ container.2 }}" - project_name: "{{ container.3 }}" - container: "{{ container.4 }}_{{ container.7 }}" - container_access: "{{ container.5 }}" + auth: "{{ container.0 }}" + container: "{{ container.1 }}_{{ container.4 }}" + container_access: "{{ container.2 }}" wait: yes - state: "{{ container.6 }}" + state: "{{ container.3 }}" with_nested: - - ["{{ endpoint }}"] - - ["{{ username }}"] - - ["{{ password }}"] - - ["{{ project }}"] + - ["{{ auth_var | provide_default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['access'] | default('private') }}"] - ["{{ state }}"] @@ -48,20 +37,13 @@ - name: "Async:: provision/deprovision swift containers when count specified" os_object: - auth: - auth_url: "{{ container.0 }}" - username: "{{ container.1 }}" - password: "{{ container.2 }}" - project_name: "{{ container.3 }}" - container: "{{ container.4 }}_{{ container.7 }}" - container_access: "{{ container.5 }}" + auth: "{{ container.0 }}" + container: "{{ container.1 }}_{{ container.4 }}" + container_access: "{{ container.2 }}" wait: yes - state: "{{ container.6 }}" + state: "{{ container.3 }}" with_nested: - - ["{{ endpoint }}"] - - ["{{ username }}"] - - ["{{ password }}"] - - ["{{ project }}"] + - ["{{ auth_var | provide_default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['access'] | default('private') }}"] - ["{{ state }}"] diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_server.yml b/linchpin/provision/roles/openstack/tasks/provision_os_server.yml index 2bf80f192..5d2bf1948 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_server.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_server.yml @@ -1,11 +1,7 @@ - name: "provision/deprovision os_server resources by looping on count" os_server2: state: "{{ state }}" - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" name: "{{ res_grp_name }}_{{ res_def['res_name'] | default(res_def['name']) }}" image: "{{ res_def['image'] }}" key_name: "{{ res_def['keypair'] }}" @@ -26,11 +22,7 @@ - name: "provision/deprovision os_server resources by looping on count" os_server2: state: "{{ state }}" - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" name: "{{ res_grp_name }}_{{ res_def['res_name'] | default(res_def['name']) }}" image: "{{ res_def['image'] }}" key_name: "{{ res_def['keypair'] }}" diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml b/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml index e6c84f5e4..95fc9705f 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml @@ -1,34 +1,23 @@ - name: "provisioning/deprovisioning of security group" os_security_group: - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" - name: "provisioning/deprovisioning of security group" os_security_group_rule: - auth: - auth_url: "{{ group.0 }}" - username: "{{ group.1 }}" - password: "{{ group.2 }}" - project_name: "{{ group.3 }}" - security_group: "{{ group.4 }}" - state: "{{ group.5 }}" - protocol: "{{ group.6['proto'] }}" - direction: "{{ group.6['rule_type'] | os_sg_rule_type }}" - port_range_min: "{{ group.6['from_port'] }}" - port_range_max: "{{ group.6['to_port'] }}" - remote_ip_prefix: "{{ group.6['cidr_ip'] }}" + auth: "{{ group.0 }}" + security_group: "{{ group.1 }}" + state: "{{ group.2 }}" + protocol: "{{ group.3['proto'] }}" + direction: "{{ group.3['rule_type'] | os_sg_rule_type }}" + port_range_min: "{{ group.3['from_port'] }}" + port_range_max: "{{ group.3['to_port'] }}" + remote_ip_prefix: "{{ group.3['cidr_ip'] }}" wait: "yes" with_nested: - - ["{{ endpoint }}"] - - ["{{ username }}"] - - ["{{ password }}"] - - ["{{ project }}"] + - ["{{ auth_var | provide_default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ state }}"] - "{{ res_def['rules'] }}" diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml b/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml index b241eacd1..c1e52a484 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml @@ -1,11 +1,7 @@ --- - name: "provision/deprovision the os volume resource type" os_volume: - auth: - auth_url: "{{ endpoint }}" - username: "{{ username }}" - password: "{{ password }}" - project_name: "{{ project }}" + auth: "{{ auth_var | provide_default(omit) }}" state: "{{ state }}" size: "{{ res_def['size'] }}" display_name: "{{ res_def['res_name'] | default(res_def['name']) }}" @@ -18,19 +14,12 @@ - name: "provision/deprovision cinder volumes when count specified" os_volume: - auth: - auth_url: "{{ vol.0 }}" - username: "{{ vol.1 }}" - password: "{{ vol.2 }}" - project_name: "{{ vol.3 }}" - state: "{{ vol.6 }}" - size: "{{ vol.5 }}" - display_name: "{{ vol.4 }}_{{ vol.7 }}" + auth: "{{ vol.0 }}" + state: "{{ vol.3 }}" + size: "{{ vol.2 }}" + display_name: "{{ vol.1 }}_{{ vol.4 }}" with_nested: - - ["{{ endpoint }}"] - - ["{{ username }}"] - - ["{{ password }}"] - - ["{{ project }}"] + - ["{{ auth_var | provide_default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['size'] }}"] - ["{{ state }}"] @@ -48,19 +37,12 @@ - name: "Async:: provision/deprovision cinder volumes when count specified" os_volume: - auth: - auth_url: "{{ vol.0 }}" - username: "{{ vol.1 }}" - password: "{{ vol.2 }}" - project_name: "{{ vol.3 }}" - state: "{{ vol.6 }}" - size: "{{ vol.5 }}" - display_name: "{{ vol.4 }}_{{ vol.7 }}" + auth: "{{ vol.0 }}" + state: "{{ vol.3 }}" + size: "{{ vol.2 }}" + display_name: "{{ vol.1 }}_{{ vol.4 }}" with_nested: - - ["{{ endpoint }}"] - - ["{{ username }}"] - - ["{{ password }}"] - - ["{{ project }}"] + - ["{{ auth_var | provide_default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['size'] }}"] - ["{{ state }}"] diff --git a/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml b/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml index 53dde6d1f..8bf32bf13 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml @@ -2,9 +2,27 @@ debug: msg: "The current server obj is {{ res_grp }} " -- name: "Including credentials of current resource {{ res_grp['resource_group_name'] }} " - include_vars: "roles/openstack/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials']) }}.yml" - no_log: false +- name: "Unset the authvar from previous run" + set_fact: + auth_var: "" + +- name: "set cred profile" + set_fact: + cred_profile: "{{ res_grp['credentials']['profile'] | default('default') }}" + +- name: "Get creds from auth driver" + auth_driver: + name: "{{ res_grp['credentials']['name'] }}" + cred_type: "openstack" + cred_path: "{{ creds_path }}" + driver: "file" + register: auth_var + ignore_errors: true + +- name: "set auth_var " + set_fact: + auth_var: "{{ auth_var['output']['clouds'][cred_profile]['auth'] | default('') }}" + ignore_errors: true - name: "provisioning resource definitions of current group" include: provision_res_defs.yml res_def={{ res_item.0 }} res_grp_name={{ res_item.1 }} From 2ffac00acbb6edbd7a6c6dbedcd5de64a7e75c2a Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:54:48 -0400 Subject: [PATCH 06/16] updated aws playbooks to incorporate auth_driver functionality --- .../roles/aws/tasks/provision_aws_cfn.yml | 8 +++---- .../roles/aws/tasks/provision_aws_ec2.yml | 8 +++---- .../roles/aws/tasks/provision_aws_ec2_key.yml | 4 ++-- .../roles/aws/tasks/provision_aws_s3.yml | 4 ++-- .../roles/aws/tasks/provision_aws_sg.yml | 4 ++-- .../aws/tasks/provision_resource_group.yml | 24 ++++++++++++++++--- .../roles/aws/tasks/teardown_aws_cfn.yml | 8 +++---- .../roles/aws/tasks/teardown_aws_ec2.yml | 8 +++---- .../roles/aws/tasks/teardown_aws_ec2_key.yml | 12 +++++----- .../roles/aws/tasks/teardown_aws_s3.yml | 4 ++-- 10 files changed, 51 insertions(+), 33 deletions(-) diff --git a/linchpin/provision/roles/aws/tasks/provision_aws_cfn.yml b/linchpin/provision/roles/aws/tasks/provision_aws_cfn.yml index fd5ba910c..289a43fea 100644 --- a/linchpin/provision/roles/aws/tasks/provision_aws_cfn.yml +++ b/linchpin/provision/roles/aws/tasks/provision_aws_cfn.yml @@ -12,8 +12,8 @@ - name: "Provision cloud formation stack" cloudformation: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" stack_name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" region: "{{ res_def['region'] }}" @@ -27,8 +27,8 @@ - name: "Provision cloud formation stack" cloudformation: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" stack_name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" region: "{{ res_def['region'] }}" diff --git a/linchpin/provision/roles/aws/tasks/provision_aws_ec2.yml b/linchpin/provision/roles/aws/tasks/provision_aws_ec2.yml index 9894db027..acd5ea92e 100644 --- a/linchpin/provision/roles/aws/tasks/provision_aws_ec2.yml +++ b/linchpin/provision/roles/aws/tasks/provision_aws_ec2.yml @@ -8,8 +8,8 @@ - name: "Provisioning AWS_EC2 Resource when not async" ec2: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" key_name: "{{ res_def['keypair'] }}" instance_type: "{{ res_def['flavor'] }}" image: "{{ res_def['image'] }}" @@ -30,8 +30,8 @@ - name: "Async:: Provisioning AWS_EC2 Resource" ec2: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" key_name: "{{ res_def['keypair'] }}" instance_type: "{{ res_def['flavor'] }}" image: "{{ res_def['image'] }}" diff --git a/linchpin/provision/roles/aws/tasks/provision_aws_ec2_key.yml b/linchpin/provision/roles/aws/tasks/provision_aws_ec2_key.yml index 50a34b4be..f63694dd6 100644 --- a/linchpin/provision/roles/aws/tasks/provision_aws_ec2_key.yml +++ b/linchpin/provision/roles/aws/tasks/provision_aws_ec2_key.yml @@ -1,7 +1,7 @@ - name: "Provisioning AWS EC2 KEY " ec2_key: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" region: "{{ res_def['region'] }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" diff --git a/linchpin/provision/roles/aws/tasks/provision_aws_s3.yml b/linchpin/provision/roles/aws/tasks/provision_aws_s3.yml index a0653aa8e..aeb8c009d 100644 --- a/linchpin/provision/roles/aws/tasks/provision_aws_s3.yml +++ b/linchpin/provision/roles/aws/tasks/provision_aws_s3.yml @@ -5,8 +5,8 @@ - name: "Provisioning AWS_S3 Resource" s3: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" bucket: "{{ res_def['res_name'] | default(res_def['name']) }}" mode: "{{ s3_mode }}" region: "{{ res_def['region'] }}" diff --git a/linchpin/provision/roles/aws/tasks/provision_aws_sg.yml b/linchpin/provision/roles/aws/tasks/provision_aws_sg.yml index ffc427886..1d0abda89 100644 --- a/linchpin/provision/roles/aws/tasks/provision_aws_sg.yml +++ b/linchpin/provision/roles/aws/tasks/provision_aws_sg.yml @@ -4,7 +4,7 @@ name: "{{ res_def['res_name'] | default(res_def['name']) }}" description: "{{ res_def['description']}}" region: "{{ res_def['region']}}" - aws_access_key: "{{ aws_access_key_id }}" - aws_secret_key: "{{ aws_secret_access_key }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] }}" rules: "{{ res_def['rules'] | aws_sg_rules('inbound') }}" rules_egress: "{{ res_def['rules'] | aws_sg_rules('outbound') }}" diff --git a/linchpin/provision/roles/aws/tasks/provision_resource_group.yml b/linchpin/provision/roles/aws/tasks/provision_resource_group.yml index e9fb41825..9819362fe 100644 --- a/linchpin/provision/roles/aws/tasks/provision_resource_group.yml +++ b/linchpin/provision/roles/aws/tasks/provision_resource_group.yml @@ -2,9 +2,27 @@ debug: msg: "The current Resource Group obj is {{ res_grp }} " -- name: "Including credentials of current resource {{ res_grp['resource_group_name'] }} " - include_vars: "roles/aws/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials']) }}.yml" - no_log: true +- name: "Unset the authvar from previous run" + set_fact: + auth_var: "" + +- name: "Set cred profile" + set_fact: + cred_profile: "{{ res_grp['credentials']['profile'] | default('default') }}" + +- name: "Get creds from auth driver" + auth_driver: + name: "{{ res_grp['credentials']['name'] }}" + cred_type: "aws" + cred_path: "{{ creds_path }}" + driver: "file" + register: auth_var + ignore_errors: true + +- name: "Set auth_var " + set_fact: + auth_var: "{{ auth_var['output'][cred_profile]['test'] | default('') }}" + ignore_errors: true - name: "provisioning resource definitions of current group" include: provision_res_defs.yml res_def={{ res_item.0 }} res_grp_name={{ res_item.1 }} diff --git a/linchpin/provision/roles/aws/tasks/teardown_aws_cfn.yml b/linchpin/provision/roles/aws/tasks/teardown_aws_cfn.yml index eda8c7ee0..e7a374e1f 100644 --- a/linchpin/provision/roles/aws/tasks/teardown_aws_cfn.yml +++ b/linchpin/provision/roles/aws/tasks/teardown_aws_cfn.yml @@ -12,8 +12,8 @@ - name: "Teardown cloud formation stack" cloudformation: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" stack_name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" region: "{{ res_def['region'] }}" @@ -28,8 +28,8 @@ - name: "Async:: Teardown cloud formation stack" cloudformation: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" stack_name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" region: "{{ res_def['region'] }}" diff --git a/linchpin/provision/roles/aws/tasks/teardown_aws_ec2.yml b/linchpin/provision/roles/aws/tasks/teardown_aws_ec2.yml index d3eac7c09..ed68283f4 100644 --- a/linchpin/provision/roles/aws/tasks/teardown_aws_ec2.yml +++ b/linchpin/provision/roles/aws/tasks/teardown_aws_ec2.yml @@ -8,8 +8,8 @@ wait: yes when: res_def_item.4 == res_def_item.5 and res_def_item.6 == "aws_ec2" and res_def_item.7 == false with_together: - - "{{ aws_access_key_id | default(omit) }}" - - "{{ aws_secret_access_key | default(omit) }}" + - "{{ auth_var['aws_access_key_id'] | default(omit) }}" + - "{{ auth_var['aws_secret_access_key'] | default(omit) }}" - "{{ tp_out.instance_ids | join(',') }}" - "{{ res_def['region'] }}" - "{{ tp_out['instances'][0]['tags']['resource_group_name'] }}" @@ -30,8 +30,8 @@ wait: yes when: res_def_item.4 == res_def_item.5 and res_def_item.6 == "aws_ec2" and res_def_item.7 == true with_together: - - "{{ aws_access_key_id | default(omit) }}" - - "{{ aws_secret_access_key | default(omit) }}" + - "{{ auth_var['aws_access_key_id'] | default(omit) }}" + - "{{ auth_var['aws_secret_access_key'] | default(omit) }}" - "{{ tp_out.instance_ids | join(',') }}" - "{{ res_def['region'] }}" - "{{ tp_out['instances'][0]['tags']['resource_group_name'] }}" diff --git a/linchpin/provision/roles/aws/tasks/teardown_aws_ec2_key.yml b/linchpin/provision/roles/aws/tasks/teardown_aws_ec2_key.yml index fb75fb3d2..9a4463bf2 100644 --- a/linchpin/provision/roles/aws/tasks/teardown_aws_ec2_key.yml +++ b/linchpin/provision/roles/aws/tasks/teardown_aws_ec2_key.yml @@ -1,9 +1,9 @@ - name: "DeProvisioning AWS_EC2_KEY Resource" ec2_key: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" - region: "{{ res_def['region'] }}" - name: "{{ res_def['res_name'] | default(res_def['name']) }}" - state: "{{ state }}" - wait: "yes" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" + region: "{{ res_def['region'] }}" + name: "{{ res_def['res_name'] | default(res_def['name']) }}" + state: "{{ state }}" + wait: "yes" when: res_def['res_type'] == 'aws_ec2_key' diff --git a/linchpin/provision/roles/aws/tasks/teardown_aws_s3.yml b/linchpin/provision/roles/aws/tasks/teardown_aws_s3.yml index c02cc60f9..740bd7457 100644 --- a/linchpin/provision/roles/aws/tasks/teardown_aws_s3.yml +++ b/linchpin/provision/roles/aws/tasks/teardown_aws_s3.yml @@ -5,8 +5,8 @@ - name: "DeProvisioning AWS_S3 Resource" s3: - aws_access_key: "{{ aws_access_key_id | default(omit) }}" - aws_secret_key: "{{ aws_secret_access_key | default(omit) }}" + aws_access_key: "{{ auth_var['aws_access_key_id'] | default(omit) }}" + aws_secret_key: "{{ auth_var['aws_secret_access_key'] | default(omit) }}" bucket: "{{ res_def['res_name'] | default(res_def['name']) }}" mode: "{{ s3_mode }}" region: "{{ res_def['region'] }}" From 84c41ec58b51d188d6e7a4318ffa9b32f64502e6 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Fri, 12 May 2017 18:55:50 -0400 Subject: [PATCH 07/16] updated gcloud playbooks to incorporate auth_driver functionality --- .../gcloud/tasks/provision_gcloud_gce.yml | 14 +++++------ .../roles/gcloud/tasks/provision_res_defs.yml | 4 ++-- .../gcloud/tasks/provision_resource_group.yml | 24 ++++++++++++++++--- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/linchpin/provision/roles/gcloud/tasks/provision_gcloud_gce.yml b/linchpin/provision/roles/gcloud/tasks/provision_gcloud_gce.yml index a4f1a8453..13d359ad6 100644 --- a/linchpin/provision/roles/gcloud/tasks/provision_gcloud_gce.yml +++ b/linchpin/provision/roles/gcloud/tasks/provision_gcloud_gce.yml @@ -1,5 +1,3 @@ - - - name: "Provision/Teardown resource by looping on count" gce2: name: "{{ res_def['res_name'] | default(res_def['name']) }}" @@ -8,9 +6,9 @@ machine_type: "{{ res_def['flavor'] }}" image: "{{ res_def['image'] }}" state: "{{ state }}" - service_account_email: "{{ client_email }}" - credentials_file: "roles/gcloud/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials']) }}.json" - project_id: "{{ project_id }}" + service_account_email: "{{ auth_var['output']['client_email'] }}" + credentials_file: "{{ auth_var['path'] }}/{{ auth_var['params']['name'] }}.json" + project_id: "{{ auth_var['output']['project_id'] }}" register: res_def_output when: not async @@ -27,9 +25,9 @@ machine_type: "{{ res_def['flavor'] }}" image: "{{ res_def['image'] }}" state: "{{ state }}" - service_account_email: "{{ client_email }}" - credentials_file: "roles/gcloud/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials']) }}.json" - project_id: "{{ project_id }}" + service_account_email: "{{ auth_var['output']['client_email'] }}" + credentials_file: "{{ auth_var['path'] }}/{{ auth_var['params']['name'] }}.json" + project_id: "{{ auth_var['output']['project_id'] }}" register: res_def_output when: async async: "{{ async_timeout }}" diff --git a/linchpin/provision/roles/gcloud/tasks/provision_res_defs.yml b/linchpin/provision/roles/gcloud/tasks/provision_res_defs.yml index b44c78b4c..62137e335 100644 --- a/linchpin/provision/roles/gcloud/tasks/provision_res_defs.yml +++ b/linchpin/provision/roles/gcloud/tasks/provision_res_defs.yml @@ -1,3 +1,3 @@ #- name: "Register resource count" -- name: "provision resource type : {{ res_def['res_type'] }}" - include: "provision_{{ res_def['res_type'] }}.yml" +- name: "provision resource type : {{ res_def['res_type'] | default(res_def['type']) }}" + include: "provision_{{ res_def['res_type'] | default(res_def['type']) }}.yml" diff --git a/linchpin/provision/roles/gcloud/tasks/provision_resource_group.yml b/linchpin/provision/roles/gcloud/tasks/provision_resource_group.yml index edc08ccf9..098e29892 100644 --- a/linchpin/provision/roles/gcloud/tasks/provision_resource_group.yml +++ b/linchpin/provision/roles/gcloud/tasks/provision_resource_group.yml @@ -2,9 +2,27 @@ debug: msg: "The current server obj is {{ res_grp }}" -- name: "Including credentials of current resource {{ res_grp['resource_group_name'] }} " - include_vars: "roles/gcloud/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials'])}}.json" - no_log: true +- name: "Unset the authvar from previous run" + set_fact: + auth_var: "" + +- name: "set cred profile" + set_fact: + cred_profile: "{{ res_grp['credentials']['profile'] | default('default') }}" + +- name: "Get creds from auth driver" + auth_driver: + name: "{{ res_grp['credentials']['name'] }}" + cred_type: "gcloud" + cred_path: "{{ creds_path }}" + driver: "file" + register: auth_var + ignore_errors: true + +- name: "set auth_var " + set_fact: + auth_var: "{{ auth_var['output'][cred_profile]['test'] | default('') }}" + ignore_errors: true - name: "provisioning resource definitions of current group" include: provision_res_defs.yml res_def={{ res_item.0 }} res_grp_name={{ res_item.1 }} From b5e1291fbb55e97504ea36ce228244199260a18a Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 15 May 2017 09:27:56 -0400 Subject: [PATCH 08/16] Updated linchpin cli , and api to fetch the creds_path variable pass it as evar to run_cli --- linchpin/__init__.py | 8 +++++++- linchpin/api/__init__.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/linchpin/__init__.py b/linchpin/__init__.py index 8505ef8eb..45a24608d 100644 --- a/linchpin/__init__.py +++ b/linchpin/__init__.py @@ -60,12 +60,15 @@ def get_command(self, ctx, name): @click.option('-w', '--workspace', type=click.Path(), envvar='WORKSPACE', help='Use the specified workspace if the familiar Jenkins $WORKSPACE environment variable ' 'is not set') +@click.option('-c', '--creds', type=click.Path(), envvar='LP_CREDS', + help='Use the specified credentials path if RKSPACE environment variable ' + 'is not set') @click.option('-v', '--verbose', is_flag=True, default=False, help='Enable verbose output') @click.option('--version', is_flag=True, help='Prints the version and exits') @pass_context -def runcli(ctx, config, workspace, verbose, version): +def runcli(ctx, config, workspace, creds, verbose, version): """linchpin: hybrid cloud orchestration""" ctx.verbose = verbose @@ -81,7 +84,10 @@ def runcli(ctx, config, workspace, verbose, version): if workspace is not None: ctx.workspace = os.path.realpath(os.path.expanduser(workspace)) + ctx.creds_path = os.path.realpath(os.path.expanduser(workspace)) + ctx.log_debug("ctx.workspace: {0}".format(ctx.workspace)) + ctx.log_debug("ctx.creds_path: {0}".format(ctx.creds_path)) ctx.pinfile = ctx.cfgs['init']['pinfile'] diff --git a/linchpin/api/__init__.py b/linchpin/api/__init__.py index 0423c3d77..5c9c1927d 100755 --- a/linchpin/api/__init__.py +++ b/linchpin/api/__init__.py @@ -188,6 +188,8 @@ def run_playbook(self, pinfile, targets='all', playbook='up'): if self.ctx.cfgs.get('ansible'): ansible_console = ast.literal_eval(self.ctx.cfgs['ansible'].get('console', 'False')) + self.ctx.evars['creds_path'] = self.ctx.creds_path + if not ansible_console: ansible_console = self.ctx.verbose From 11af8b4d765a7b7db31981b761a2319fdee5f3e6 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 15:52:37 -0400 Subject: [PATCH 09/16] Renamed the parameter --creds to --creds-path --- linchpin/__init__.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/linchpin/__init__.py b/linchpin/__init__.py index 45a24608d..68db4e77d 100644 --- a/linchpin/__init__.py +++ b/linchpin/__init__.py @@ -60,15 +60,15 @@ def get_command(self, ctx, name): @click.option('-w', '--workspace', type=click.Path(), envvar='WORKSPACE', help='Use the specified workspace if the familiar Jenkins $WORKSPACE environment variable ' 'is not set') -@click.option('-c', '--creds', type=click.Path(), envvar='LP_CREDS', - help='Use the specified credentials path if RKSPACE environment variable ' - 'is not set') @click.option('-v', '--verbose', is_flag=True, default=False, help='Enable verbose output') @click.option('--version', is_flag=True, help='Prints the version and exits') +@click.option('-cp', '--creds-path', type=click.Path(), envvar='LP_CREDS', + help='Use the specified credentials path if WORKSPACE environment variable ' + 'is not set') @pass_context -def runcli(ctx, config, workspace, creds, verbose, version): +def runcli(ctx, config, workspace, verbose, version, creds_path): """linchpin: hybrid cloud orchestration""" ctx.verbose = verbose @@ -83,8 +83,13 @@ def runcli(ctx, config, workspace, creds, verbose, version): if workspace is not None: ctx.workspace = os.path.realpath(os.path.expanduser(workspace)) + else: + ctx.workspace = os.getenv('PWD') - ctx.creds_path = os.path.realpath(os.path.expanduser(workspace)) + if creds_path is not None: + ctx.creds_path = os.path.realpath(os.path.expanduser(creds_path)) + else: + ctx.creds_path = None ctx.log_debug("ctx.workspace: {0}".format(ctx.workspace)) ctx.log_debug("ctx.creds_path: {0}".format(ctx.creds_path)) From df29db986c177e34dde4e26b239424e66106f168 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 16:36:43 -0400 Subject: [PATCH 10/16] bugfix: Make provide default return the fetched items --- linchpin/provision/filter_plugins/provide_default.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/linchpin/provision/filter_plugins/provide_default.py b/linchpin/provision/filter_plugins/provide_default.py index 8f615684e..b6eec1802 100755 --- a/linchpin/provision/filter_plugins/provide_default.py +++ b/linchpin/provision/filter_plugins/provide_default.py @@ -4,6 +4,8 @@ def provide_default(fetched, default): if fetched == "": return default + else: + return fetched class FilterModule(object): ''' A filter to fix interface's name format ''' From 48d165db58008bfe0082d5fc7368bd3823beb24f Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 17:25:24 -0400 Subject: [PATCH 11/16] Updated schema for aws resource group type --- linchpin/defaults/schemas/schema_v4.json | 25 ++++++++++++------------ 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/linchpin/defaults/schemas/schema_v4.json b/linchpin/defaults/schemas/schema_v4.json index d07b2e40e..1604cff0d 100644 --- a/linchpin/defaults/schemas/schema_v4.json +++ b/linchpin/defaults/schemas/schema_v4.json @@ -9,18 +9,6 @@ "description": "Date of creation", "type": "string" }, - "site": { - "description": "site of provisioning ", - "type": "string" - }, - "credentials": { - "description": "contains various credential_vaults names", - "type": "array", - "items":{ - "type":"string" - }, - "minimum":1 - }, "resource_groups": { "description": "contains list of resources ", "type": "array", @@ -747,7 +735,18 @@ }, "credentials": { "description":"contains creds file associated to this resource", - "type":"string" + "type":"object", + "properties": { + "profile": { + "type": "string" + }, + "auth_type": { + "type": "string" + }, + "name": { + "type": "string" + } + } } }, "required":["resource_group_name","resource_group_type","resource_definitions","credentials"], From 810f991a113355eb5fce5c76f4290ff95aa8e885 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 17:25:56 -0400 Subject: [PATCH 12/16] Updated Authdriver references inside aws role The auth_var is now used across all the playbooks to refer the aws_access_id and aws_secret_key --- linchpin/provision/roles/aws/tasks/main.yml | 2 +- .../aws/tasks/provision_resource_group.yml | 2 +- .../aws/tasks/teardown_resource_group.yml | 23 ++++++++++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/linchpin/provision/roles/aws/tasks/main.yml b/linchpin/provision/roles/aws/tasks/main.yml index 47819cbfb..d023a214b 100644 --- a/linchpin/provision/roles/aws/tasks/main.yml +++ b/linchpin/provision/roles/aws/tasks/main.yml @@ -22,5 +22,5 @@ include: teardown_resource_group.yml res_grp={{ item.0 }} topo_output_file={{ item.1 }} with_nested: - "{{ aws_res_grps }}" - - ["{{ resources_file | default( default_resources_path+'/'+outputs.topology_name+'.output.yaml' ) }}"] + - ["{{ resources_file | default( default_resources_path+'/'+outputs.topology_name+'.output' ) }}"] when: state == "absent" diff --git a/linchpin/provision/roles/aws/tasks/provision_resource_group.yml b/linchpin/provision/roles/aws/tasks/provision_resource_group.yml index 9819362fe..a2f67381f 100644 --- a/linchpin/provision/roles/aws/tasks/provision_resource_group.yml +++ b/linchpin/provision/roles/aws/tasks/provision_resource_group.yml @@ -21,7 +21,7 @@ - name: "Set auth_var " set_fact: - auth_var: "{{ auth_var['output'][cred_profile]['test'] | default('') }}" + auth_var: "{{ auth_var['output'][cred_profile] | default('') }}" ignore_errors: true - name: "provisioning resource definitions of current group" diff --git a/linchpin/provision/roles/aws/tasks/teardown_resource_group.yml b/linchpin/provision/roles/aws/tasks/teardown_resource_group.yml index ddbf3a78a..41057de51 100644 --- a/linchpin/provision/roles/aws/tasks/teardown_resource_group.yml +++ b/linchpin/provision/roles/aws/tasks/teardown_resource_group.yml @@ -2,18 +2,29 @@ debug: msg: "The current server obj is {{ res_grp }}" -- name: "Including credentials of current resource {{ res_grp['resource_group_name'] }}" - include_vars: "roles/aws/vars/{{ res_grp['assoc_creds'] | default(res_grp['credentials']) }}.yml" - no_log: true +- name: "Set cred profile" + set_fact: + cred_profile: "{{ res_grp['credentials']['profile'] | default('default') }}" + +- name: "Get creds from auth driver" + auth_driver: + name: "{{ res_grp['credentials']['name'] }}" + cred_type: "aws" + cred_path: "{{ creds_path }}" + driver: "file" + register: auth_var + ignore_errors: true + +- name: "Set auth_var " + set_fact: + auth_var: "{{ auth_var['output'][cred_profile] | default('') }}" + ignore_errors: true - name: "Parsing topology output file" output_parser: output_file: "{{ topo_output_file }}" register: topo_output -#- name: "teardown AWS ec2 resources" -# include: teardown_res_defs.yml -# when: state == "absent" # patch for res_type to type translation - name: "Add attribute res_type to res_grp resource_definitions" From 80655fbc7e2c1bb794b17902b1a43c7569d5f277 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 17:35:30 -0400 Subject: [PATCH 13/16] Updated setting for creds_path evar param using set_evar --- linchpin/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linchpin/api/__init__.py b/linchpin/api/__init__.py index 5c9c1927d..bea1dc26c 100755 --- a/linchpin/api/__init__.py +++ b/linchpin/api/__init__.py @@ -188,7 +188,7 @@ def run_playbook(self, pinfile, targets='all', playbook='up'): if self.ctx.cfgs.get('ansible'): ansible_console = ast.literal_eval(self.ctx.cfgs['ansible'].get('console', 'False')) - self.ctx.evars['creds_path'] = self.ctx.creds_path + self.set_evar('creds_path', self.ctx.creds_path) if not ansible_console: ansible_console = self.ctx.verbose From 2d7176d39924423248444d8e3fd58967892e2c68 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 18:54:53 -0400 Subject: [PATCH 14/16] Added multiple tasks to handle auth_var when not given, as its not possible to unset a var in ansible --- .../openstack/tasks/provision_os_keypair.yml | 26 ++++++++-- .../openstack/tasks/provision_os_object.yml | 52 +++++++++++++++++-- .../openstack/tasks/provision_os_server.yml | 39 ++++++++++++-- .../roles/openstack/tasks/provision_os_sg.yml | 34 ++++++++++-- .../openstack/tasks/provision_os_volume.yml | 51 +++++++++++++++--- .../tasks/provision_resource_group.yml | 2 +- 6 files changed, 182 insertions(+), 22 deletions(-) diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml b/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml index 757d748d2..1a57f6b4b 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_keypair.yml @@ -1,11 +1,19 @@ - name: "provisioning/deprovisioning keypair" os_keypair: - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" register: res_def_output - when: not async + when: not async and auth_var['changed']== true + +- name: "provisioning/deprovisioning keypair" + os_keypair: + name: "{{ res_def['res_name'] | default(res_def['name']) }}" + state: "{{ state }}" + wait: "yes" + register: res_def_output + when: not async and auth_var['changed']== false - name: "Generate keyfile " copy: @@ -20,14 +28,24 @@ - name: "Async:: provisioning/deprovisioning keypair" os_keypair: - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" register: res_def_output async: "{{ async_timeout }}" poll: 0 - when: async + when: async and auth_var['changed']== true + +- name: "Async:: provisioning/deprovisioning keypair" + os_keypair: + name: "{{ res_def['res_name'] | default(res_def['name']) }}" + state: "{{ state }}" + wait: "yes" + register: res_def_output + async: "{{ async_timeout }}" + poll: 0 + when: async and auth_var['changed']== false - name: 'Async:: Check on keypair create task' async_status: jid={{ res_def_output.ansible_job_id }} diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_object.yml b/linchpin/provision/roles/openstack/tasks/provision_os_object.yml index 00da759cd..40cb0bc56 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_object.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_object.yml @@ -5,12 +5,20 @@ - name: "provision/deprovision swift containers" os_object: - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" container: "{{ res_def['res_name'] | default(res_def['name']) }}" container_access: "{{ res_def['access'] | default('private') }}" wait: yes state: "{{ state }}" - when: res_def['count'] is not defined + when: res_def['count'] is not defined and auth_var['changed'] == true + +- name: "provision/deprovision swift containers" + os_object: + container: "{{ res_def['res_name'] | default(res_def['name']) }}" + container_access: "{{ res_def['access'] | default('private') }}" + wait: yes + state: "{{ state }}" + when: res_def['count'] is not defined and auth_var['changed'] == false - name: "provision/deprovision swift containers when count specified" os_object: @@ -20,7 +28,7 @@ wait: yes state: "{{ container.3 }}" with_nested: - - ["{{ auth_var | provide_default(omit) }}"] + - ["{{ auth_var | default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['access'] | default('private') }}"] - ["{{ state }}"] @@ -28,7 +36,23 @@ loop_control: loop_var: container register: res_def_output - when: res_def['count'] is defined and not async + when: res_def['count'] is defined and not async and auth_var['changed'] == true + +- name: "provision/deprovision swift containers when count specified" + os_object: + container: "{{ container.1 }}_{{ container.3 }}" + container_access: "{{ container.1 }}" + wait: yes + state: "{{ container.2 }}" + with_nested: + - ["{{ res_def['res_name'] | default(res_def['name']) }}"] + - ["{{ res_def['access'] | default('private') }}"] + - ["{{ state }}"] + - "{{ res_count.stdout }}" + loop_control: + loop_var: container + register: res_def_output + when: res_def['count'] is defined and not async and auth_var['changed'] == false - name: "Append outputitem to topology_outputs" set_fact: @@ -53,7 +77,25 @@ async: "{{ async_timeout }}" poll: 0 register: res_def_output - when: res_def['count'] is defined and async + when: res_def['count'] is defined and async and auth_var['changed'] == true + +- name: "Async:: provision/deprovision swift containers when count specified" + os_object: + container: "{{ container.0 }}_{{ container.3 }}" + container_access: "{{ container.1 }}" + wait: yes + state: "{{ container.2 }}" + with_nested: + - ["{{ res_def['res_name'] | default(res_def['name']) }}"] + - ["{{ res_def['access'] | default('private') }}"] + - ["{{ state }}"] + - "{{ res_count.stdout }}" + loop_control: + loop_var: container + async: "{{ async_timeout }}" + poll: 0 + register: res_def_output + when: res_def['count'] is defined and async and auth_var['changed'] == false #- name: "Async: save the job id" # set_fact: diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_server.yml b/linchpin/provision/roles/openstack/tasks/provision_os_server.yml index 5d2bf1948..49ed8e2c6 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_server.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_server.yml @@ -1,7 +1,7 @@ - name: "provision/deprovision os_server resources by looping on count" os_server2: state: "{{ state }}" - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" name: "{{ res_grp_name }}_{{ res_def['res_name'] | default(res_def['name']) }}" image: "{{ res_def['image'] }}" key_name: "{{ res_def['keypair'] }}" @@ -11,7 +11,22 @@ floating_ip_pools: "{{ res_def['fip_pool'] | default(omit) }}" security_groups: "{{ res_def['security_groups'] | default(omit) }}" count: "{{ res_def['count'] }}" - when: not async + when: not async and auth_var['changed'] == true + register: res_def_output + +- name: "provision/deprovision os_server resources by looping on count" + os_server2: + state: "{{ state }}" + name: "{{ res_grp_name }}_{{ res_def['res_name'] | default(res_def['name']) }}" + image: "{{ res_def['image'] }}" + key_name: "{{ res_def['keypair'] }}" + api_timeout: 99999 + flavor: "{{ res_def['flavor'] }}" + nics: "{{ res_def['networks'] | os_net }}" + floating_ip_pools: "{{ res_def['fip_pool'] | default(omit) }}" + security_groups: "{{ res_def['security_groups'] | default(omit) }}" + count: "{{ res_def['count'] }}" + when: not async and auth_var['changed'] == false register: res_def_output - name: "Append outputitem to topology_outputs" @@ -35,7 +50,25 @@ async: "{{ async_timeout | default(1000) }}" poll: 0 register: res_def_output - when: async + when: async and auth_var['changed'] == true + +- name: "provision/deprovision os_server resources by looping on count" + os_server2: + state: "{{ state }}" + name: "{{ res_grp_name }}_{{ res_def['res_name'] | default(res_def['name']) }}" + image: "{{ res_def['image'] }}" + key_name: "{{ res_def['keypair'] }}" + api_timeout: 99999 + flavor: "{{ res_def['flavor'] }}" + nics: "{{ res_def['networks'] | os_net }}" + floating_ip_pools: "{{ res_def['fip_pool'] | default(omit) }}" + security_groups: "{{ res_def['security_groups'] | default(omit) }}" + count: "{{ res_def['count'] }}" + async: "{{ async_timeout | default(1000) }}" + poll: 0 + register: res_def_output + when: async and auth_var['changed'] == false + # following tasks saves the async job details - name: "Async:: save the job id" diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml b/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml index 95fc9705f..317baa212 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_sg.yml @@ -1,9 +1,17 @@ - name: "provisioning/deprovisioning of security group" os_security_group: - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" name: "{{ res_def['res_name'] | default(res_def['name']) }}" state: "{{ state }}" wait: "yes" + when: auth_var['changed'] == true + +- name: "provisioning/deprovisioning of security group" + os_security_group: + name: "{{ res_def['res_name'] | default(res_def['name']) }}" + state: "{{ state }}" + wait: "yes" + when: auth_var['changed'] == false - name: "provisioning/deprovisioning of security group" os_security_group_rule: @@ -17,14 +25,34 @@ remote_ip_prefix: "{{ group.3['cidr_ip'] }}" wait: "yes" with_nested: - - ["{{ auth_var | provide_default(omit) }}"] + - ["{{ auth_var | default(omit) }}"] - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ state }}"] - "{{ res_def['rules'] }}" loop_control: loop_var: group register: res_def_output - when: state == "present" + when: state == "present" and auth_var['changed'] == true + +- name: "provisioning/deprovisioning of security group" + os_security_group_rule: + security_group: "{{ group.0 }}" + state: "{{ group.1 }}" + protocol: "{{ group.2['proto'] }}" + direction: "{{ group.2['rule_type'] | os_sg_rule_type }}" + port_range_min: "{{ group.2['from_port'] }}" + port_range_max: "{{ group.2['to_port'] }}" + remote_ip_prefix: "{{ group.2['cidr_ip'] }}" + wait: "yes" + with_nested: + - ["{{ res_def['res_name'] | default(res_def['name']) }}"] + - ["{{ state }}"] + - "{{ res_def['rules'] }}" + loop_control: + loop_var: group + register: res_def_output + when: state == "present" and auth_var['changed'] == false + - name: "Append outputitem to topology_outputs" set_fact: diff --git a/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml b/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml index c1e52a484..2bf68d8f2 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_os_volume.yml @@ -1,11 +1,18 @@ --- - name: "provision/deprovision the os volume resource type" os_volume: - auth: "{{ auth_var | provide_default(omit) }}" + auth: "{{ auth_var | default(omit) }}" state: "{{ state }}" size: "{{ res_def['size'] }}" display_name: "{{ res_def['res_name'] | default(res_def['name']) }}" - when: res_def['count'] is not defined + when: res_def['count'] is not defined and auth_var['changed'] == true + +- name: "provision/deprovision the os volume resource type" + os_volume: + state: "{{ state }}" + size: "{{ res_def['size'] }}" + display_name: "{{ res_def['res_name'] | default(res_def['name']) }}" + when: res_def['count'] is not defined and auth_var['changed'] == false - name: "Register resource count" shell: python -c "print [x for x in range( 0, {{ res_def['count'] }} )]" @@ -19,7 +26,22 @@ size: "{{ vol.2 }}" display_name: "{{ vol.1 }}_{{ vol.4 }}" with_nested: - - ["{{ auth_var | provide_default(omit) }}"] + - ["{{ auth_var | default(omit) }}"] + - ["{{ res_def['res_name'] | default(res_def['name']) }}"] + - ["{{ res_def['size'] }}"] + - ["{{ state }}"] + - "{{ res_count.stdout }}" + loop_control: + loop_var: vol + register: res_def_output + when: res_def['count'] is defined and not async and auth_var['changed'] == true + +- name: "provision/deprovision cinder volumes when count specified" + os_volume: + state: "{{ vol.2 }}" + size: "{{ vol.1 }}" + display_name: "{{ vol.0 }}_{{ vol.3 }}" + with_nested: - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['size'] }}"] - ["{{ state }}"] @@ -27,7 +49,7 @@ loop_control: loop_var: vol register: res_def_output - when: res_def['count'] is defined and not async + when: res_def['count'] is defined and not async and auth_var['changed'] == false - name: "Append outputitem to topology_outputs" set_fact: @@ -42,7 +64,24 @@ size: "{{ vol.2 }}" display_name: "{{ vol.1 }}_{{ vol.4 }}" with_nested: - - ["{{ auth_var | provide_default(omit) }}"] + - ["{{ auth_var | default(omit) }}"] + - ["{{ res_def['res_name'] | default(res_def['name']) }}"] + - ["{{ res_def['size'] }}"] + - ["{{ state }}"] + - "{{ res_count.stdout }}" + loop_control: + loop_var: vol + async: "{{ async_timeout }}" + poll: 0 + register: res_def_output + when: res_def['count'] is defined and async and auth_var['changed'] == true + +- name: "Async:: provision/deprovision cinder volumes when count specified" + os_volume: + state: "{{ vol.2 }}" + size: "{{ vol.1 }}" + display_name: "{{ vol.0 }}_{{ vol.3 }}" + with_nested: - ["{{ res_def['res_name'] | default(res_def['name']) }}"] - ["{{ res_def['size'] }}"] - ["{{ state }}"] @@ -52,7 +91,7 @@ async: "{{ async_timeout }}" poll: 0 register: res_def_output - when: res_def['count'] is defined and async + when: res_def['count'] is defined and async and auth_var['changed'] == false #following tasks saves the async job details - name: "Async:: save the job id" diff --git a/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml b/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml index 8bf32bf13..bb10388c5 100644 --- a/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml +++ b/linchpin/provision/roles/openstack/tasks/provision_resource_group.yml @@ -21,7 +21,7 @@ - name: "set auth_var " set_fact: - auth_var: "{{ auth_var['output']['clouds'][cred_profile]['auth'] | default('') }}" + auth_var: "{{ auth_var['output']['clouds'][cred_profile]['auth'] | default(omit) }}" ignore_errors: true - name: "provisioning resource definitions of current group" From ee7fd2fedb0edb155bfe9ba80d144be0092898b8 Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Mon, 22 May 2017 18:56:42 -0400 Subject: [PATCH 15/16] Added creds_path to linchpin api context and defaulted it to None --- linchpin/__init__.py | 2 +- linchpin/api/context.py | 1 + linchpin/linchpin.conf | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/linchpin/__init__.py b/linchpin/__init__.py index 68db4e77d..42010198a 100644 --- a/linchpin/__init__.py +++ b/linchpin/__init__.py @@ -89,7 +89,7 @@ def runcli(ctx, config, workspace, verbose, version, creds_path): if creds_path is not None: ctx.creds_path = os.path.realpath(os.path.expanduser(creds_path)) else: - ctx.creds_path = None + ctx.creds_path = str(None) ctx.log_debug("ctx.workspace: {0}".format(ctx.workspace)) ctx.log_debug("ctx.creds_path: {0}".format(ctx.creds_path)) diff --git a/linchpin/api/context.py b/linchpin/api/context.py index e48f77d90..25a4b42c0 100644 --- a/linchpin/api/context.py +++ b/linchpin/api/context.py @@ -40,6 +40,7 @@ def __init__(self): self.lib_path = os.path.realpath(os.path.join(lib_path, os.pardir)) self.workspace = os.path.realpath(os.path.curdir) + self.creds_path = None def load_config(self, lpconfig=None): diff --git a/linchpin/linchpin.conf b/linchpin/linchpin.conf index 315b6baec..83a631d28 100644 --- a/linchpin/linchpin.conf +++ b/linchpin/linchpin.conf @@ -23,7 +23,7 @@ async = False async_timeout = 1000 output = True check_mode = False - +creds_path = None # default paths in playbooks # # lp_path = /linchpin From 0fd98c7f9609a8ffa2f8186b0edb331c121b28ba Mon Sep 17 00:00:00 2001 From: samvarankashyap Date: Tue, 23 May 2017 11:38:56 -0400 Subject: [PATCH 16/16] bugfix: Updated gcloud schema , added name, auth_type to credentials attribute --- linchpin/defaults/schemas/schema_v4.json | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/linchpin/defaults/schemas/schema_v4.json b/linchpin/defaults/schemas/schema_v4.json index 1604cff0d..0e4729680 100644 --- a/linchpin/defaults/schemas/schema_v4.json +++ b/linchpin/defaults/schemas/schema_v4.json @@ -773,7 +773,15 @@ }, "credentials": { "description":"contains creds file associated to this resource", - "type":"string" + "type":"object", + "properties": { + "auth_type": { + "type": "string" + }, + "name": { + "type": "string" + } + } } }, "required":["resource_group_name","resource_group_type","resource_definitions","credentials"],