Add integration test to check linchpin.latest output

Check linchpin.latest output to make sure that the data we receive
from the various providers is up to date

Signed-off-by: Ryan Cole <rycole@redhat.com>

config/Dockerfiles/tests.d/inventory/02_correct_data

@@ -0,0 +1,71 @@
+#!/bin/bash -xe
+
+# Verify basic provisioning for all supplied providers
+# distros.exclude: none
+# providers.include: none
+# providers.exclude: gce, ovirt
+
+DISTRO=${1}
+PROVIDER=${2}
+
+TARGET="${PROVIDER}-new"
+KEY=""
+TEST_NAME="${DISTRO}/${PROVIDER}/01_general"
+DESCRIPTION="Test ${PROVIDER} provider with a simple up/destroy"
+
+echo "Test Name: ${TEST_NAME}"
+echo "${DESCRIPTION}"
+
+
+
+if [ "${PROVIDER}" == "libvirt" ]; then
+    TARGET="libvirt-new"
+    KEY="libvirt_res"
+elif [ "${PROVIDER}" == "openstack" ]; then
+    TARGET="os-server-new os-sg-new"
+    KEY="os_server_res"
+elif [ "${PROVIDER}" == "aws" ]; then
+    TARGET="aws-ec2-new"
+    KEY="aws_ec2_res"
+elif [ "${PROVIDER}" == "beaker" ]; then
+    TARGET="beaker-new"
+    KEY="beaker_res"
+elif [ "${PROVIDER}" == "duffy" ]; then
+    TARGET="duffy-new"
+    KEY="duffy_res"
+else
+    TARGET="dummy-new"
+    KEY="dummy_res"
+fi
+
+pushd docs/source/examples/workspaces/${PROVIDER}
+
+function clean_up {
+    set +e
+    D_RC=0
+    for target in ${TARGETS}; do
+        linchpin -w . -v destroy ${target}
+        D_RC=(${?} -o ${D_RC})
+    done
+    if [ ${D_RC} -ne 0 ]; then
+        exit ${D_RC}
+    fi
+}
+trap clean_up EXIT SIGHUP SIGINT SIGTERM
+
+echo "CREDS_PATH: ${CREDS_PATH}"
+linchpin -w . -vvvv up ${TARGET}
+
+popd
+file1=config/Dockerfiles/tests.d/inventories/benchmarks/${PROVIDER}.latest
+file2=docs/source/examples/workspaces/${PROVIDER]/resources/linchpin.latest
+
+python compare_dicts.py $file1 $file2 $key
+
+RC=${?}
+
+if [ ${RC} -eq 0 ]; then
+    exit 0
+else
+    exit 1
+fi

config/Dockerfiles/tests.d/inventory/benchmark-data/aws.latest

@@ -0,0 +1 @@
+{"4": {"action": "up", "targets": [{"aws-ec2-new": {"outputs": {"inventory_path": ["/home/ryan/Development/Python/linchpin/docs/source/examples/workspaces/aws/inventories/ec2-new-362700.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [{"failed": false, "changed": true, "tagged_instances": [], "instance_ids": ["i-0b6a94638c1452283"], "instances": [{"kernel": null, "root_device_type": "ebs", "private_dns_name": "ip-10-178-241-7.ec2.internal", "public_ip": "54.92.144.136", "private_ip": "10.178.241.7", "id": "i-0b6a94638c1452283", "ebs_optimized": false, "state": "running", "virtualization_type": "hvm", "root_device_name": "/dev/sda1", "ramdisk": null, "block_device_mapping": {"/dev/sdb": {"status": "attached", "delete_on_termination": true, "volume_id": "vol-09c9c57f5d23bf7f6"}, "/dev/sda1": {"status": "attached", "delete_on_termination": true, "volume_id": "vol-0a2bf247736f50160"}}, "key_name": null, "image_id": "ami-984189e2", "tenancy": "default", "groups": {"sg-eae64983": "default", "sg-8a1d78e3": "public"}, "public_dns_name": "ec2-54-92-144-136.compute-1.amazonaws.com", "state_code": 16, "tags": {"color": "blue", "resource_group_name": "aws", "shape": "oval", "name": "demo-day"}, "placement": "us-east-1d", "ami_launch_index": "0", "dns_name": "ec2-54-92-144-136.compute-1.amazonaws.com", "region": "us-east-1", "launch_time": "2018-11-09T15:43:00.000Z", "instance_type": "m1.small", "architecture": "x86_64", "hypervisor": "xen"}]}], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "7": {"uhash": "362700", "rc": 0}, "inputs": {"topology_data": {"topology_name": "ec2-new", "resource_groups": [{"resource_group_name": "aws", "resource_definitions": [{"count": 1, "name": "demo-day", "region": "us-east-1", "role": "aws_ec2", "image": "ami-984189e2", "security_group": ["default", "public"], "flavor": "m1.small", "instance_tags": {"color": "blue", "shape": "oval"}}], "resource_group_type": "aws", "credentials": {"profile": "default", "filename": "aws.key"}}]}}}}]}}

config/Dockerfiles/tests.d/inventory/benchmark-data/beaker.latest

@@ -0,0 +1 @@
+{"6": {"action": "up", "targets": [{"beaker-new": {"outputs": {"inventory_path": ["/home/ryan/Development/Python/linchpin/docs/source/examples/workspaces/beaker/inventories/bkr-new-368089.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [], "beaker_res": [{"status": "Running", "name": "rhel65", "family": "RedHatEnterpriseLinux6", "url": "https://beaker.engineering.redhat.com/jobs/3078684", "variant": "Client", "system": "hp-dl380pgen8-02-vm-14.lab.bos.redhat.com", "result": "Pass", "rid": "6095054", "arch": "x86_64", "id": "3078684", "distro": "RHEL-6.5"}], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "inputs": {"topology_data": {"topology_name": "bkr-new", "resource_groups": [{"resource_group_name": "bkr-new", "resource_definitions": [{"cancel_message": "Job cancelled on account of rain", "whiteboard": "Provisioned with linchpin", "recipesets": [{"count": 1, "arch": "x86_64", "name": "rhel65", "distro": "RHEL-6.5"}], "role": "bkr_server", "attempt_wait_time": 60, "max_attempts": 240}], "resource_group_type": "beaker"}]}}, "6": {"uhash": "368089", "rc": 0}}}]}}

config/Dockerfiles/tests.d/inventory/benchmark-data/dummy.latest

@@ -0,0 +1 @@
+{"7": {"action": "up", "targets": [{"dummy-new": {"outputs": {"inventory_path": ["/home/ryan/Development/Python/linchpin/docs/source/examples/workspaces/dummy/inventories/dummy_cluster-7d4ffc.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["web-7d4ffc-0.example.net", "web-7d4ffc-1.example.net", "web-7d4ffc-2.example.net"], "dummy_file": "/tmp/dummy.hosts"}, {"failed": false, "changed": true, "hosts": ["test-7d4ffc-0.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "7": {"uhash": "7d4ffc", "rc": 0}, "inputs": {"layout_data": {"inventory_layout": {"inventory_file": "{{ workspace }}/inventories/dummy-new-{{ uhash }}.inventory", "hosts": [{"count": 3, "name": "example-node", "host_groups": ["example"]}, {"count": 1, "name": "test-node", "host_groups": ["test"]}], "vars": {"hostname": "__IP__"}, "host_groups": {"all": {"vars": {"ansible_private_key_file": "\"{{ lookup('env', 'TESTLP') | default('/tmp', true) }}/CSS/keystore/css-central\"\n", "ansible_user": "root"}}}}}, "topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_definitions": [{"count": 3, "role": "dummy_node", "name": "web"}, {"count": 1, "role": "dummy_node", "name": "test"}], "resource_group_type": "dummy"}]}}}, "dummy-topo": {"outputs": {"inventory_path": ["/home/ryan/Development/Python/linchpin/docs/source/examples/workspaces/dummy/inventories/dummy_cluster-4f2a68.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["web-4f2a68-0.example.net", "web-4f2a68-1.example.net", "web-4f2a68-2.example.net"], "dummy_file": "/tmp/dummy.hosts"}, {"failed": false, "changed": true, "hosts": ["test-4f2a68-0.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "4": {"uhash": "4f2a68", "rc": 0}, "inputs": {"layout_data": {"inventory_layout": {"hosts": [{"count": 3, "name": "example-node", "host_groups": ["example"]}, {"count": 1, "name": "test-node", "host_groups": ["test"]}], "vars": {"hostname": "__IP__"}}}, "topology_data": {"topology_name": "dummy_cluster", "resource_groups": [{"resource_group_name": "dummy", "resource_definitions": [{"count": 3, "role": "dummy_node", "name": "web"}, {"count": 1, "role": "dummy_node", "name": "test"}], "resource_group_type": "dummy"}]}}}}]}}

config/Dockerfiles/tests.d/inventory/benchmark-data/libvirt.latest

@@ -0,0 +1 @@
+{"1": {"action": "up", "targets": [{"libvirt-new": {"1": {"uhash": "775dc1", "rc": 0}, "outputs": {"inventory_path": ["/workdir/docs/source/examples/workspaces/libvirt/inventories/libvirt-new.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_key_res": [], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [], "beaker_res": [], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [{"ip": "192.168.132.196", "name": "centos71_0"}], "os_volume_res": []}}, "inputs": {"layout_data": {"inventory_layout": {"hosts": [{"count": 1, "name": "example-node", "host_groups": ["example"]}], "vars": {"hostname": "__IP__"}}}, "topology_data": {"topology_name": "libvirt-new", "resource_groups": [{"resource_group_name": "libvirt-new", "resource_definitions": [{"count": 1, "cloud_config": {"users": [{"name": "herlo", "sudo": "ALL=(ALL) NOPASSWD:ALL", "lock_passwd": true, "gecos": "Clint Savage", "groups": "wheel", "ssh_import_id": "gh:herlo"}]}, "name": "centos71", "ssh_key": "libvirt", "uri": "qemu:///system", "vcpus": 1, "additional_storage": "10G", "role": "libvirt_node", "memory": 2048, "arch": "x86_64", "networks": [{"name": "default"}], "image_src": "http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1608.qcow2.xz"}], "resource_group_type": "libvirt"}]}}}}]}}

config/Dockerfiles/tests.d/inventory/benchmark-data/openshift.latest

@@ -0,0 +1 @@
+{"8": {"action": "up", "targets": [{"openshift-new": {"8": {"uhash": "e4579e", "rc": 2}, "inputs": {"topology_data": {"topology_name": "openshift", "resource_groups": [{"resource_group_name": "test1", "resource_definitions": [{"role": "openshift_inline", "data": [{"kind": "ReplicationController", "spec": {"replicas": 7, "template": {"spec": {"restartPolicy": "Always", "containers": [{"image": "redhatqecinch/jenkins_slave:latest", "name": "jenkins-slave", "env": [{"name": "JENKINS_MASTER_URL", "value": "http://10.8.172.6/"}, {"name": "JSLAVE_NAME", "value": "mynode"}]}], "securityPolicy": {"runAsUser": 1000090000}}, "metadata": {"labels": {"name": "jenkins-slave"}}}, "selector": {"name": "jenkins-slave"}}, "apiVersion": "v1", "metadata": {"namespace": "central-ci-test-ghelling", "name": "jenkins-slave"}}], "name": "openshift"}], "resource_group_type": "openshift", "credentials": {"api_endpoint": "192.168.42.206:8443/", "api_token": "7Pz4dMNSxvD7-JipQfYDEYRCV-bdd9Una18ye3E3jHw"}}]}}}}]}}

config/Dockerfiles/tests.d/inventory/compare_dicts.py

@@ -0,0 +1,35 @@
+import sys
+import pprint
+import json
+from deepdiff import DeepDiff
+
+
+def find_item(obj, key):
+    if key in obj:
+        return obj[key]
+    for k, v in obj.items():
+        if isinstance(v, dict):
+            item = finditem(v, key)
+            if item is not None:
+                return item
+
+
+expected = sys.argv[1]
+test = sys.argv[2]
+key = sys.argv[3]
+
+expected_data = json.loads(open(expected))
+test_data = json.loads(open(test))
+
+expected_returns = find_item(expected_data, key)
+test_returns = find_item(test_data, key)
+
+diff = DeepDiff(expected_returns, test_returns)
+if 'values_changed' in diff.keys():
+    diff.pop('values_changed', None)
+
+if diff != {}:
+    pprint(diff, indent=2)
+    sys.exit(1)
+
+sys.exit(0)

config/Dockerfiles/tests.d/inventory/linchpin.benchmark

@@ -0,0 +1 @@
+{"27": {"action": "up", "targets": [{"complex-inventory": {"outputs": {"inventory_path": ["/home/ryan/Development/Python/linchpin/linchpin/tests/mockdata/complex-inventory/inventories/dummy-aws-5c6dff.inventory"], "resources": {"os_keypair_res": [], "rax_server_res": [], "aws_ec2_key_res": [], "os_server_res": [], "ovirt_vms_res": [], "aws_ec2_res": [{"failed": false, "changed": true, "tagged_instances": [], "instance_ids": ["i-0e3589510c0b6a0c9"], "instances": [{"kernel": null, "root_device_type": "ebs", "private_dns_name": "ip-10-146-230-134.ec2.internal", "public_ip": "54.80.207.97", "private_ip": "10.146.230.134", "id": "i-0e3589510c0b6a0c9", "ebs_optimized": false, "state": "running", "virtualization_type": "hvm", "root_device_name": "/dev/sda1", "ramdisk": null, "block_device_mapping": {"/dev/sdb": {"status": "attached", "delete_on_termination": true, "volume_id": "vol-0ed39bcd714f90515"}, "/dev/sda1": {"status": "attached", "delete_on_termination": true, "volume_id": "vol-08c48c43b5d372ecf"}}, "key_name": null, "image_id": "ami-984189e2", "tenancy": "default", "groups": {"sg-eae64983": "default", "sg-8a1d78e3": "public"}, "public_dns_name": "ec2-54-80-207-97.compute-1.amazonaws.com", "state_code": 16, "tags": {"color": "blue", "resource_group_name": "aws-topology", "shape": "oval", "name": "demo-day"}, "placement": "us-east-1d", "ami_launch_index": "0", "dns_name": "ec2-54-80-207-97.compute-1.amazonaws.com", "region": "us-east-1", "launch_time": "2018-11-08T14:47:35.000Z", "instance_type": "m1.small", "architecture": "x86_64", "hypervisor": "xen"}]}], "gcloud_gce_res": [], "aws_s3_res": [], "duffy_res": [], "os_sg_res": [], "dummy_res": [{"failed": false, "changed": true, "hosts": ["test-5c6dff-0.example.net", "test-5c6dff-1.example.net", "test-5c6dff-2.example.net"], "dummy_file": "/tmp/dummy.hosts"}], "beaker_res": [{"status": "Running", "name": "rhel75-1", "family": "RedHatEnterpriseLinux7", "url": "https://beaker.engineering.redhat.com/jobs/3074615", "variant": "Server", "system": "intel-wildcatpass-03.khw1.lab.eng.bos.redhat.com", "result": "Pass", "rid": "6089530", "arch": "x86_64", "id": "3074615", "distro": "RHEL-7.4"}, {"status": "Running", "name": "rhel75-1", "family": "RedHatEnterpriseLinux7", "url": "https://beaker.engineering.redhat.com/jobs/3074615", "variant": "Server", "system": "hp-ml150gen9-01.rhts.eng.bos.redhat.com", "result": "Pass", "rid": "6089531", "arch": "x86_64", "id": "3074615", "distro": "RHEL-7.5"}, {"status": "Running", "name": "rhel75-2", "family": "RedHatEnterpriseLinux7", "url": "https://beaker.engineering.redhat.com/jobs/3074615", "variant": "Server", "system": "intel-wildcatpass-03.khw1.lab.eng.bos.redhat.com", "result": "Pass", "rid": "6089530", "arch": "x86_64", "id": "3074615", "distro": "RHEL-7.4"}, {"status": "Running", "name": "rhel75-2", "family": "RedHatEnterpriseLinux7", "url": "https://beaker.engineering.redhat.com/jobs/3074615", "variant": "Server", "system": "hp-ml150gen9-01.rhts.eng.bos.redhat.com", "result": "Pass", "rid": "6089531", "arch": "x86_64", "id": "3074615", "distro": "RHEL-7.5"}], "aws_cfn_res": [], "os_heat_res": [], "os_obj_res": [], "libvirt_res": [], "os_volume_res": []}}, "30": {"uhash": "5c6dff", "rc": 0}, "inputs": {"layout_data": {"inventory_layout": {"hosts": [{"count": 2, "name": "dummy-node", "host_groups": ["dummy-layout"]}, {"count": 1, "name": "dummy-master", "host_groups": ["dummy-layout", "dummy-master"]}, {"count": 1, "name": "aws-node", "host_groups": ["aws-layout"]}, {"count": 1, "name": "beaker-node", "host_groups": ["beaker-layout"]}], "vars": {"hostname": "__IP__"}, "host_groups": {"dummy": {"children": ["dummy-layout", "dummy-master"], "vars": {"ansible_ssh_common_args": "-o StrictHostKeyChecking=no -4"}}, "all": {"vars": {"ansible_ssh_common_args": "-o StrictHostKeyChecking=no", "ansible_python_interpreter": "/usr/bin/python3", "ansible_user": "root"}}}}}, "topology_data": {"topology_name": "dummy-aws", "resource_groups": [{"resource_group_name": "dummy-topology", "resource_definitions": [{"count": 3, "role": "dummy_node", "name": "test"}], "resource_group_type": "dummy"}, {"resource_group_name": "aws-topology", "resource_definitions": [{"count": 1, "instance_tags": {"color": "blue", "shape": "oval"}, "name": "demo-day", "region": "us-east-1", "role": "aws_ec2", "security_group": ["default", "public"], "flavor": "m1.small", "image": "ami-984189e2"}], "resource_group_type": "aws", "credentials": {"profile": "default", "filename": "aws.key"}}, {"resource_group_name": "beaker-topology", "resource_definitions": [{"cancel_message": "Job cancelled on account of rain", "whiteboard": "Provisioned with linchpin", "recipesets": [{"count": 1, "name": "rhel75-1", "variant": "Server", "bkr_data": "lvm-on", "hostrequires": [{"tag": "arch", "value": "X86_64", "op": "="}, {"tag": "memory", "value": "1000", "op": ">="}], "keyvalue": ["DISKSPACE>=100000"], "arch": "x86_64", "distro": "RHEL-7.4"}, {"count": 1, "name": "rhel75-2", "variant": "Server", "keyvalue": ["DISKSPACE>=100000"], "arch": "x86_64", "distro": "RHEL-7.5"}], "role": "bkr_server", "attempt_wait_time": 60, "max_attempts": 240}], "resource_group_type": "beaker"}]}}}}]}}