Move inventory files into roles

Signed-off-by: 14rcole <rycole@redhat.com>

docs/source/examples/workspaces/linchpin.conf

@@ -115,8 +115,8 @@ debug_mode = True
 
 # the uhash value will still exist, but will not be added to
 # instances or the inventory_path
-#enable_uhash = False
-enable_uhash = True
+enable_uhash = False
+#enable_uhash = True
 
 # in older versions of linchpin (<v1.0.4), a resources folder exists, which
 # dumped the data that is now stored in the RunDB. To disable the resources

linchpin/FilterUtils/FilterUtils.py

@@ -4,10 +4,11 @@
 from six.moves import range
 
 
-def add_res_type(hosts, res_type):
+def add_res_data(hosts, res_grp, role):
     new_hosts = []
     for host in hosts:
-        host['resource_type'] = res_type
+        host['resource_group'] = res_grp
+        host['role'] = role
         new_hosts.append(host)
     return new_hosts
 
@@ -96,7 +97,7 @@ def get_host_from_uri(uri):
 def get_provider_resources(topo_output, res_type):
     provider_resources = []
     for host in topo_output:
-        if host['resource_type'] == res_type:
+        if host['resource_group'] == res_type:
             provider_resources.append(host)
     return provider_resources
 
@@ -278,7 +279,7 @@ def fetch_beaker_job_ids(topo_out):
 def get_os_server_names(topo_output):
     names = []
     for item in topo_output:
-        if item["resource_type"] == "os_server_res":
+        if item["role"] == "os_server":
             openstack_res = item.get("openstack", [])
             for os_item in openstack_res:
                 names.append(os_item["name"])

linchpin/InventoryFilters/GenericInventory.py

@@ -1,30 +1,52 @@
 #!/usr/bin/env python
 
+import imp
+import os
+
 from .InventoryFilter import InventoryFilter
-from .InventoryProviders import get_all_drivers
 from .InventoryProviders import get_inv_formatter
 
 
 class GenericInventory(InventoryFilter):
 
-    def __init__(self, inv_format="cfg"):
+    def __init__(self, inv_format="cfg", pb_path=None):
         InventoryFilter.__init__(self)
-        self.filter_classes = get_all_drivers()
+        self.pb_path = pb_path
         self.inv_formatter = get_inv_formatter(inv_format)()
 
     def get_host_data(self, res_output, config):
         """
         """
         host_data = []
         for res_grp in res_output:
-            inv_filter = res_grp['resource_type']
+            res_type = res_grp['resource_group']
             # check only when the inventory_filter exists
-            if inv_filter in list(self.filter_classes.keys()):
-                data = self.filter_classes[inv_filter]()\
-                    .get_host_data(res_grp, config)
-                host_data.append(data)
+            filter_class = self.get_filter_class(res_type)
+            data = filter_class.get_host_data(res_grp, config)
+            host_data.append(data)
         return host_data
 
+
+    def get_filter_class(self, res_type):
+        path = "{0}/files/inventory.py".format(self._find_role_path(res_type))
+        provider = imp.load_source('Inventory', path)
+        return provider.Inventory()
+
+
+    def _find_role_path(self, res_type):
+        """
+        returns the full path to the given playbook
+
+        :params res_type: name of the role
+        """
+
+        for path in self.pb_path:
+            p = '{0}/{1}/{2}'.format(path, 'roles', res_type)
+
+            if os.path.exists(os.path.expanduser(p)):
+                return p
+
+
     def get_hosts_by_count(self, host_data, count):
         """
         currently this function gets all the ips/hostname according to the

linchpin/InventoryFilters/InventoryProviders.py

@@ -3,60 +3,16 @@
 """
 from __future__ import absolute_import
 from linchpin.exceptions import LinchpinError
-
-from .AWSInventory import AWSInventory
-from .BeakerInventory import BeakerInventory
-from .DuffyInventory import DuffyInventory
-from .DummyInventory import DummyInventory
-from .GCloudInventory import GCloudInventory
-from .LibvirtInventory import LibvirtInventory
-from .OpenstackInventory import OpenstackInventory
-from .OvirtInventory import OvirtInventory
-from .VMwareInventory import VMwareInventory
-from .DockerInventory import DockerInventory
-from .AzureInventory import AzureInventory
-from .OpenshiftInventory import OpenshiftInventory
-
 from .JSONInventoryFormatter import JSONInventoryFormatter
 from .CFGInventoryFormatter import CFGInventoryFormatter
 
-filter_classes = {
-    "azure_vm": AzureInventory,
-    "aws_ec2_res": AWSInventory,
-    "beaker_res": BeakerInventory,
-    "duffy_res": DuffyInventory,
-    "dummy_res": DummyInventory,
-    "nummy_res": DummyInventory,
-    "gcloud_gce_res": GCloudInventory,
-    "libvirt_res": LibvirtInventory,
-    "os_server_res": OpenstackInventory,
-    "ovirt_vms_res": OvirtInventory,
-    "vmware_guest_res": VMwareInventory,
-    "docker_container_res": DockerInventory,
-    "docker_image_res": DockerInventory,
-    "openshift_res": OpenshiftInventory
-}
-
 formatter_classes = {
     "cfg": CFGInventoryFormatter,
     "ini": CFGInventoryFormatter,
     "json": JSONInventoryFormatter
 }
 
 
-def get_driver(provider):
-
-    if provider not in filter_classes:
-        raise LinchpinError("Key {0} not found in"
-                            " inventory provider dict".format(provider))
-
-    return filter_classes[provider]
-
-
-def get_all_drivers():
-    return filter_classes
-
-
 def get_inv_formatter(inv_type):
     if inv_type not in formatter_classes:
         raise LinchpinError("Key {0} not found in"

linchpin/__init__.py

@@ -370,7 +370,8 @@ def _convert_layout(self, layout_data):
 
     def generate_inventory(self, resource_data, layout, inv_format="cfg",
                            topology_data={}, config_data={}):
-        inv = GenericInventory.GenericInventory(inv_format=inv_format)
+        inv = GenericInventory.GenericInventory(inv_format=inv_format,
+                                                pb_path=self.pb_path)
         inventory = inv.get_inventory(resource_data, layout, topology_data,
                                       config_data)
         return inventory

linchpin/provision/filter_plugins/add_res_type.py → linchpin/provision/filter_plugins/add_res_data.py

@@ -3,8 +3,8 @@
 
 
 class FilterModule(object):
-    ''' A filter to add_res_type '''
+    ''' A filter to add_res_data '''
     def filters(self):
         return {
-            'add_res_type': filter_utils.add_res_type
+            'add_res_data': filter_utils.add_res_data
         }

linchpin/InventoryFilters/AWSInventory.py → linchpin/provision/roles/aws/files/inventory.py

@@ -3,10 +3,10 @@
 from __future__ import absolute_import
 from collections import OrderedDict
 
-from .InventoryFilter import InventoryFilter
+from linchpin.InventoryFilters.InventoryFilter import InventoryFilter
 
 
-class AWSInventory(InventoryFilter):
+class Inventory(InventoryFilter):
     DEFAULT_HOSTNAMES = ['public_dns_name', 'public_ip', 'private_ip']
 
     def get_host_data(self, res, cfgs):
@@ -15,28 +15,24 @@ def get_host_data(self, res, cfgs):
         inventory file, based on available data. Only a single hostname or IP
         address will be returned per instance, so as to avoid duplicate runs of
         Ansible on the same host via the generated inventory file.
-
         Each hostname contains mappings of any variable that was defined in the
         cfgs section of the PinFile (e.g. __IP__) to the value in the field that
         corresponds with that variable in the cfgs.
-
         If an instance has a public IP attached, its hostname in DNS will be
         returned if available, and if not the public IP address will be used.
         For instances which have a private IP address for VPC use cases, the
         private IP address will be returned since private EC2 hostnames (e.g.
         ip-10-0-0-1.ec2.internal) will not typically be resolvable outside of
         AWS. For instances with both a public and private IP address, the
         public address is always returned instead of the private address.
-
         :param topo:
             linchpin AWS EC2 resource data
-
         :param cfgs:
             map of config options from PinFile
         """
 
         host_data = OrderedDict()
-        if res['resource_type'] != 'aws_ec2_res':
+        if res['resource_group'] != 'aws' or res['role'] != 'aws_ec2':
             return host_data
         var_data = cfgs.get('aws', {})
         if var_data is None:

linchpin/provision/roles/aws/tasks/provision_aws_cfn.yml

@@ -47,4 +47,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_cfn: "{{ topology_outputs_aws_cfn | add_res_type('aws_cf_res') }}"
+    topology_outputs_aws_cfn: "{{ topology_outputs_aws_cfn | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2.yml

@@ -61,9 +61,9 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_ec2 | add_res_type( 'aws_ec2_res') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_ec2 | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
 
 - name: "Async:: Add type to resource"
   set_fact:
-    async_outputs_aws_ec2: "{{ async_outputs_aws_ec2 | add_res_type( 'aws_ec2_res') }}"
+    async_outputs_aws_ec2: "{{ async_outputs_aws_ec2 | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: _async

linchpin/provision/roles/aws/tasks/provision_aws_ec2_eip.yml

@@ -25,4 +25,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type( 'aws_ec2_eip') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_elb_lb.yml

@@ -48,4 +48,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type( 'aws_ec2_elb_lb') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_key.yml

@@ -22,4 +22,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2_key: "{{ topology_outputs_aws_ec2_key | add_res_type('aws_ec2_key_res') }}"
+    topology_outputs_aws_ec2_key: "{{ topology_outputs_aws_ec2_key | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_endpoint.yml

@@ -59,5 +59,5 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type('aws_ec2_vpc_endpoint') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: state == 'present'

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_internet_gateway.yml

@@ -33,4 +33,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_internet_output | add_res_type( 'aws_ec2_vpc_internet_gateway') }}"    
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_internet_output | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}" 

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_nat_gateway.yml

@@ -62,4 +62,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type( 'aws_ec2_vpc_nat_gateway') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_net.yml

@@ -26,4 +26,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type( 'aws_ec2_eip') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_routetable.yml

@@ -47,4 +47,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type('aws_ec2_vpc_routetable') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name')) }}"

linchpin/provision/roles/aws/tasks/provision_aws_ec2_vpc_subnet.yml

@@ -50,4 +50,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type('aws_ec2_vpc_subnet') }}"
+    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/provision_aws_s3.yml

@@ -41,4 +41,4 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_aws_s3: "{{ topology_outputs_aws_s3 | add_res_type('aws_s3_res') }}"
+    topology_outputs_aws_s3: "{{ topology_outputs_aws_s3 | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"

linchpin/provision/roles/aws/tasks/teardown_aws_ec2_vpc_endpoint.yml

@@ -69,8 +69,3 @@
   with_items: "{{ vpc_endpoint_ids_fetched['vpc_endpoints'] | default([]) }}"
   loop_control:
     loop_var: ep_id
-
-- name: "Add type to resource"
-  set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type('aws_ec2_vpc_endpoint') }}"
-  when: state == 'present'

linchpin/provision/roles/aws/tasks/teardown_aws_ec2_vpc_nat_gateway.yml

@@ -43,7 +43,3 @@
   set_fact:
     topology_outputs_aws_net: "{{ topology_outputs_aws_net + [res_def_output] }}"
   when: res_def_output['changed'] == true
-
-- name: "Add type to resource"
-  set_fact:
-    topology_outputs_aws_ec2: "{{ topology_outputs_aws_net | add_res_type( 'aws_ec2_vpc_nat_gateway') }}"

linchpin/InventoryFilters/AzureInventory.py → linchpin/provision/roles/azure/files/inventory.py

@@ -1,10 +1,10 @@
 from __future__ import absolute_import
 from collections import OrderedDict
 
-from .InventoryFilter import InventoryFilter
+from linchpin.InventoryFilters.InventoryFilter import InventoryFilter
 
 
-class AzureInventory(InventoryFilter):
+class Inventory(InventoryFilter):
     # DEFAULT_HOSTNAMES = ['public_dns_name', 'public_ip', 'private_ip']
 
     def get_host_data(self, res, cfgs):
@@ -26,7 +26,7 @@ def get_host_data(self, res, cfgs):
         """
 
         host_data = OrderedDict()
-        if res['resource_type'] != 'azure_vm':
+        if res['resource_group'] != 'azure'or res['role'] != 'azure_vm':
             return host_data
         networks = res['properties']['networkProfile']['networkInterfaces']
         for network in networks:

linchpin/provision/roles/azure/tasks/provision_azure_resource_group.yml

@@ -37,10 +37,10 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_azure_res_grp: "{{ topology_outputs_azure_res_grp | add_res_type( 'azure_res_grp') }}"
+    topology_outputs_azure_res_grp: "{{ topology_outputs_azure_res_grp | add_res_data(lookup('vars', 'role_name')m res_def['role']) }}"
 
 
 - name: "Async:: Add type to resource"
   set_fact:
-    async_outputs_azure_res_grp: "{{ async_outputs_azure_res_grp | add_res_type( 'azure_res_grp') }}"
+    async_outputs_azure_res_grp: "{{ async_outputs_azure_res_grp | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: _async

linchpin/provision/roles/azure/tasks/provision_azure_virtual_network.yml

@@ -42,10 +42,10 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_azure_vn: "{{ topology_outputs_azure_vn | add_res_type( 'azure_virtual_network') }}"
+    topology_outputs_azure_vn: "{{ topology_outputs_azure_vn | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
 
 
 - name: "Async:: Add type to resource"
   set_fact:
-    async_outputs_azure_vn: "{{ async_outputs_azure_vn | add_res_type( 'azure_virtual_network') }}"
+    async_outputs_azure_vn: "{{ async_outputs_azure_vn | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: _async

linchpin/provision/roles/azure/tasks/provision_azure_virtual_subnet.yml

@@ -49,10 +49,10 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_azure_vn_subnet: "{{ topology_outputs_azure_vn_subnet | add_res_type( 'azure_virtual_subnet') }}"
+    topology_outputs_azure_vn_subnet: "{{ topology_outputs_azure_vn_subnet | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
 
 
 - name: "Async:: Add type to resource"
   set_fact:
-    async_outputs_azure_vn_subnet: "{{ async_outputs_azure_vn_subnet | add_res_type( 'azure_virtual_subnet') }}"
+    async_outputs_azure_vn_subnet: "{{ async_outputs_azure_vn_subnet | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: _async

linchpin/provision/roles/azure/tasks/provision_azure_vm.yml

@@ -113,10 +113,10 @@
 
 - name: "Add type to resource"
   set_fact:
-    topology_outputs_azure_vm: "{{ topology_outputs_azure_vm | add_res_type( 'azure_vm') }}"
+    topology_outputs_azure_vm: "{{ topology_outputs_azure_vm | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
 
 
 - name: "Async:: Add type to resource"
   set_fact:
-    async_outputs_azure_vm: "{{ async_outputs_azure_vm | add_res_type( 'azure_vm') }}"
+    async_outputs_azure_vm: "{{ async_outputs_azure_vm | add_res_data(lookup('vars', 'role_name'), res_def['role']) }}"
   when: _async