From 7a262145659ab71ca27dc79fc8fa995a4fba40fc Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Mon, 16 Mar 2020 07:23:15 -0400
Subject: [PATCH 01/10] changing instance, instance type and accout tables in
 cloud realm to have single column auto increment key instead of a composite
 increment key

---
 .../Cloud/GroupBys/GroupByConfiguration.php   | 25 ++++++++++---------
 .../Query/Cloud/GroupBys/GroupByProject.php   | 20 +++++++--------
 .../etl.d/xdmod-migration-8_5_1-9_0_0.json    | 20 +++++++++++++++
 .../cloud-migration-8-5-1_9-0-0.sql           | 21 ++++++++++++++++
 .../etl_tables.d/cloud_common/account.json    |  1 -
 .../etl_tables.d/cloud_common/instance.json   |  7 ------
 .../cloud_common/instance_type.json           |  7 ------
 tests/ci/bootstrap.sh                         | 11 +++++++-
 tests/ci/scripts/xdmod-upgrade-cloud.tcl      | 12 ++++++++-
 9 files changed, 84 insertions(+), 40 deletions(-)
 create mode 100644 configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql

diff --git a/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByConfiguration.php b/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByConfiguration.php
index 5e25582dc7..e471bd4004 100644
--- a/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByConfiguration.php
+++ b/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByConfiguration.php
@@ -26,18 +26,19 @@ public function __construct()
             'configuration',
             array(),
             'SELECT DISTINCT
-                gt.id,
+                gt.display as id,
                 gt.instance_type AS short_name,
                 gt.display AS long_name,
                 gt.instance_type_id AS order_id
             FROM instance_type gt
             WHERE 1
-            ORDER BY order_id'
+            ORDER BY order_id',
+            array()
         );
-        $this->_id_field_name = 'instance_type_id';
+        $this->_id_field_name = 'display';
         $this->_long_name_field_name = 'display';
-        $this->_short_name_field_name = 'instance_type';
-        $this->_order_id_field_name = 'instance_type_id';
+        $this->_short_name_field_name = 'display';
+        $this->_order_id_field_name = 'display';
         $this->modw_schema = new Schema('modw_cloud');
         $this->configuration_table = new Table($this->modw_schema, 'instance_type', 'p');
     }
@@ -58,9 +59,9 @@ public function applyTo(Query &$query, Table $data_table, $multi_group = false)
 
         $query->addGroup($configurationtable_id_field);
 
+        $instance_type_table_id_field = new TableField($this->configuration_table, 'instance_type_id');
         $datatable_configuration_id_field = new TableField($data_table, 'instance_type_id');
-        $query->addWhereCondition(new WhereCondition($configurationtable_id_field, '=', $datatable_configuration_id_field));
-        $query->addWhereCondition(new WhereCondition(new TableField($this->configuration_table, 'resource_id'), '=', new TableField($data_table, 'host_resource_id')));
+        $query->addWhereCondition(new WhereCondition($instance_type_table_id_field, '=', $datatable_configuration_id_field));
 
         $this->addOrder($query, $multi_group);
     }
@@ -70,12 +71,12 @@ public function addWhereJoin(Query &$query, Table $data_table, $multi_group, $op
         // construct the join between the main data_table and this group by table
         $query->addTable($this->configuration_table);
 
-        $configurationtable_id_field = new TableField($this->configuration_table, $this->_id_field_name);
+        $instance_type_table_id_field = new TableField($this->configuration_table, 'instance_type_id');
         $datatable_configuration_id_field = new TableField($data_table, 'instance_type_id');
 
         // the where condition that specifies the join of the tables
-        $query->addWhereCondition(new WhereCondition($configurationtable_id_field, '=', $datatable_configuration_id_field));
-        $query->addWhereCondition(new WhereCondition(new TableField($this->configuration_table, 'resource_id'), '=', new TableField($data_table, 'host_resource_id')));
+        $query->addWhereCondition(new WhereCondition($instance_type_table_id_field, '=', $datatable_configuration_id_field));
+
         // the where condition that specifies the constraint on the joined table
         if (is_array($whereConstraint)) {
             $whereConstraint = '(' . implode(',', $whereConstraint) . ')';
@@ -106,14 +107,14 @@ public function addOrder(
 
     public function pullQueryParameters(&$request)
     {
-        return parent::pullQueryParameters2($request, '_filter_', 'instance_type_id');
+        return parent::pullQueryParameters2($request, 'SELECT DISTINCT instance_type_id FROM modw_cloud.instance_type WHERE display IN (_filter_)', 'instance_type_id');
     }
 
     public function pullQueryParameterDescriptions(&$request)
     {
         return parent::pullQueryParameterDescriptions2(
             $request,
-            'SELECT display AS field_label FROM modw_cloud.instance_type WHERE instance_type_id IN (_filter_) ORDER BY instance_type_id'
+            'SELECT display AS field_label FROM modw_cloud.instance_type WHERE display IN (_filter_) ORDER BY instance_type_id'
         );
     }
 }
diff --git a/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByProject.php b/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByProject.php
index 18598ec257..f426c2c94b 100644
--- a/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByProject.php
+++ b/classes/DataWarehouse/Query/Cloud/GroupBys/GroupByProject.php
@@ -26,16 +26,16 @@ public function __construct()
             'project',
             array(),
             'SELECT distinct
-                gt.account_id,
+                gt.display as id,
                 gt.display as short_name,
-                gt.provider_account as long_name
+                gt.display as long_name
             FROM account gt
             WHERE 1
             ORDER BY gt.ACCOUNT_ID'
         );
-        $this->_id_field_name = 'account_id';
+        $this->_id_field_name = 'display';
         $this->_long_name_field_name = 'display';
-        $this->_short_name_field_name = 'provider_account';
+        $this->_short_name_field_name = 'display';
         $this->_order_id_field_name = 'display';
         $this->modw_schema = new Schema('modw_cloud');
         $this->account_table = new Table($this->modw_schema, 'account', 'acc');
@@ -57,12 +57,10 @@ public function applyTo(Query &$query, Table $data_table, $multi_group = false)
 
         $query->addGroup($accounttable_id_field);
 
+        $account_table_id_field = new TableField($this->account_table, 'account_id');
         $datatable_account_id_field = new TableField($data_table, 'account_id');
-        $datatable_host_resource_id_field = new TableField($data_table, 'host_resource_id');
-        $accounttable_resource_id_field = new TableField($this->account_table, 'resource_id');
 
-        $query->addWhereCondition(new WhereCondition($accounttable_id_field, '=', $datatable_account_id_field));
-        $query->addWhereCondition(new WhereCondition($accounttable_resource_id_field, '=', $datatable_host_resource_id_field));
+        $query->addWhereCondition(new WhereCondition($account_table_id_field, '=', $datatable_account_id_field));
 
         $this->addOrder($query, $multi_group);
     }
@@ -72,7 +70,7 @@ public function addWhereJoin(Query &$query, Table $data_table, $multi_group, $op
         // construct the join between the main data_table and this group by table
         $query->addTable($this->account_table);
 
-        $accounttable_id_field = new TableField($this->account_table, $this->_id_field_name);
+        $accounttable_id_field = new TableField($this->account_table, 'account_id');
         $datatable_account_id_field = new TableField($data_table, 'account_id');
 
         // the where condition that specifies the join of the tables
@@ -113,14 +111,14 @@ public function addOrder(
 
     public function pullQueryParameters(&$request)
     {
-        return parent::pullQueryParameters2($request, '_filter_', 'account_id');
+        return parent::pullQueryParameters2($request, 'SELECT DISTINCT account_id FROM modw_cloud.account WHERE display in (_filter_)', 'account_id');
     }
 
     public function pullQueryParameterDescriptions(&$request)
     {
         return parent::pullQueryParameterDescriptions2(
             $request,
-            'SELECT display AS field_label FROM modw_cloud.account WHERE account_id IN (_filter_) ORDER BY display'
+            'SELECT display AS field_label FROM modw_cloud.account WHERE display IN (_filter_) ORDER BY display'
         );
     }
 }
diff --git a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
index f3ae81f1ba..2082d7ef5e 100644
--- a/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
+++ b/configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json
@@ -1,6 +1,24 @@
 {
     "module": "xdmod",
     "migration-8_5_1-9_0_0": [
+        {
+            "name": "DeleteCloudAutoincrementColumns",
+            "description": "Post-ingest updates for massaging Open Stack data",
+            "namespace": "ETL\\Maintenance",
+            "options_class": "MaintenanceOptions",
+            "class": "ExecuteSql",
+            "sql_file_list": [
+                "migrations/cloud-migration-8-5-1_9-0-0.sql"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Cloud Realm Tables",
+                    "config": "database",
+                    "schema": "modw_cloud"
+                }
+            }
+        },
         {
             "name": "update-modw_cloud-tables",
             "description": "Update modw_cloud tables",
@@ -15,6 +33,8 @@
                 "cloud_openstack/raw_event.json",
                 "cloud_common/session_records.json",
                 "cloud_common/event.json",
+                "cloud_common/account.json",
+                "cloud_common/instance_type.json",
                 "cloud_common/instance.json"
             ],
             "endpoints": {
diff --git a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
new file mode 100644
index 0000000000..dfeae21ca6
--- /dev/null
+++ b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
@@ -0,0 +1,21 @@
+ALTER TABLE modw_cloud.account MODIFY account_id INT NOT NULL;
+ALTER TABLE modw_cloud.account DROP INDEX autoincrement_key;
+ALTER TABLE modw_cloud.account DROP COLUMN account_id;
+//
+ALTER TABLE modw_cloud.instance_type MODIFY instance_type_id INT NOT NULL;
+ALTER TABLE modw_cloud.instance_type DROP INDEX increment_key;
+ALTER TABLE modw_cloud.instance_type DROP COLUMN instance_type_id;
+//
+ALTER TABLE modw_cloud.instance MODIFY instance_id INT(11) NOT NULL;
+ALTER TABLE modw_cloud.instance DROP INDEX increment_key;
+ALTER TABLE modw_cloud.instance ADD COLUMN new_instance_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.event AS ev
+JOIN
+  modw_cloud.instance AS i
+ON
+  ev.resource_id = i.resource_id AND ev.instance_id = i.instance_id
+SET
+  ev.instance_id = i.new_instance_id;
+ALTER TABLE modw_cloud.instance DROP COLUMN instance_id;
+ALTER TABLE modw_cloud.instance CHANGE new_instance_id instance_id int(11) unsigned;
diff --git a/configuration/etl/etl_tables.d/cloud_common/account.json b/configuration/etl/etl_tables.d/cloud_common/account.json
index b89f3f448a..93f9d813c7 100644
--- a/configuration/etl/etl_tables.d/cloud_common/account.json
+++ b/configuration/etl/etl_tables.d/cloud_common/account.json
@@ -55,7 +55,6 @@
             {
                 "name": "autoincrement_key",
                 "columns": [
-                    "resource_id",
                     "account_id"
                 ],
                 "is_unique": true
diff --git a/configuration/etl/etl_tables.d/cloud_common/instance.json b/configuration/etl/etl_tables.d/cloud_common/instance.json
index 37e7cab0c5..19c66c2ece 100644
--- a/configuration/etl/etl_tables.d/cloud_common/instance.json
+++ b/configuration/etl/etl_tables.d/cloud_common/instance.json
@@ -62,15 +62,8 @@
                 "is_unique": true
             },
             {
-                "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a",
-                "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column",
-                "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is",
-                "#": "useful when you want to put data into ordered groups.",
-                "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)",
-                "#": "https://www.ryadel.com/en/mysql-two-columns-primary-key-with-auto-increment/",
                 "name": "increment_key",
                 "columns": [
-                    "resource_id",
                     "instance_id"
                 ],
                 "is_unique": true
diff --git a/configuration/etl/etl_tables.d/cloud_common/instance_type.json b/configuration/etl/etl_tables.d/cloud_common/instance_type.json
index f7d4e39e16..fbef1eccd5 100644
--- a/configuration/etl/etl_tables.d/cloud_common/instance_type.json
+++ b/configuration/etl/etl_tables.d/cloud_common/instance_type.json
@@ -89,15 +89,8 @@
                 "is_unique": true
             },
             {
-                "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a",
-                "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column",
-                "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is",
-                "#": "useful when you want to put data into ordered groups.",
-                "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)",
-
                 "name": "increment_key",
                 "columns": [
-                    "resource_id",
                     "instance_type_id"
                 ],
                 "is_unique": true
diff --git a/tests/ci/bootstrap.sh b/tests/ci/bootstrap.sh
index e7786bdc9e..79252d589b 100755
--- a/tests/ci/bootstrap.sh
+++ b/tests/ci/bootstrap.sh
@@ -72,6 +72,7 @@ then
     then
         sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
         sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
+        #sudo -u xdmod php /data/xdmod/tools/etl/etl_overseer.php -p ingest-cloud-resource-specs -d "CLOUD_RESOURCE_SPECS_DIRECTORY=$REF_DIR/openstack_resource_specs" -r openstack -v debug
     fi
     sudo -u xdmod xdmod-ingestor
 
@@ -93,6 +94,14 @@ fi
 
 if [ "$XDMOD_TEST_MODE" = "upgrade" ];
 then
+    if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
+      ~/bin/services start
+      expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
+      sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
+      sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
+      sudo -u xdmod xdmod-ingestor
+      ~/bin/services stop
+    fi
     yum -y install ~/rpmbuild/RPMS/*/*.rpm
     ~/bin/services start
 
@@ -119,7 +128,7 @@ then
 
     #
     if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
-        expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
+        #expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
 
         sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
         sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
diff --git a/tests/ci/scripts/xdmod-upgrade-cloud.tcl b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
index 74c0242f72..8090d5ec13 100644
--- a/tests/ci/scripts/xdmod-upgrade-cloud.tcl
+++ b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
@@ -3,6 +3,14 @@
 # XDMoD instance. This script will fail if run against an already installed
 # XDMoD.
 
+proc confirmUpgrade { } {
+    expect {
+        timeout { send_user "\nFailed to get prompt\n"; exit 1 }
+        -re "\nDo you want to continue .*\\\] "
+    }
+    send yes\n
+}
+
 #-------------------------------------------------------------------------------
 # Configuration settings for the XDMoD resources
 
@@ -21,9 +29,11 @@ source [file join [file dirname [info script]] helper-functions.tcl]
 # they should typically not be changed as they need to match up with the
 # settings in the docker container
 
-set timeout 240
+set timeout 180
 spawn "xdmod-setup"
 
+#answerQuestion {Do you want to continue (yes, no)? [no] } Yes
+confirmUpgrade
 # Enter config settings for each resource
 selectMenuOption 4
 foreach resource $resources {

From 80766db88937c78c85e4f5e5c7f562b99bf195a7 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Mon, 16 Mar 2020 11:10:57 -0400
Subject: [PATCH 02/10] updates to configuration and project group bys

---
 .../datawarehouse.d/ref/Cloud-group-bys.json  | 34 ++++++++-----------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/configuration/datawarehouse.d/ref/Cloud-group-bys.json b/configuration/datawarehouse.d/ref/Cloud-group-bys.json
index 9f104aa6f7..f2b62a5f84 100644
--- a/configuration/datawarehouse.d/ref/Cloud-group-bys.json
+++ b/configuration/datawarehouse.d/ref/Cloud-group-bys.json
@@ -2,13 +2,6 @@
     "configuration": {
         "name": "Instance Type",
         "description_html": "The instance type of a virtual machine.",
-        "additional_join_constraints": [
-            {
-                "attribute_expr": "resource_id",
-                "operation": "=",
-                "aggregate_expr": "host_resource_id"
-            }
-        ],
         "attribute_table_schema": "modw_cloud",
         "attribute_table": "instance_type",
         "attribute_to_aggregate_table_key_map": [
@@ -19,10 +12,10 @@
         "order": 6,
         "attribute_values_query": {
             "records": {
-                "id": "instance_type_id",
-                "short_name": "instance_type",
+                "id": "display",
+                "short_name": "display",
                 "name": "display",
-                "order_id": "instance_type_id"
+                "order_id": "display"
             },
             "joins": [
                 {
@@ -33,7 +26,11 @@
                 "instance_type_id"
             ],
             "query_hint": "DISTINCT"
-        }
+        },
+        "attribute_filter_map_query": {
+            "instance_type_id": "SELECT instance_type_id FROM modw_cloud.instance_type WHERE display IN (__filter_values__)"
+        },
+        "attribute_description_query": "SELECT DISTINCT display AS filter_name FROM modw_cloud.instance_type WHERE display IN (__filter_values__) ORDER BY display"
     },
     "domain": {
         "alternate_group_by_columns": [
@@ -115,15 +112,12 @@
         "name": "User"
     },
     "project": {
-        "additional_join_constraints": [
-            {
-                "attribute_expr": "resource_id",
-                "operation": "=",
-                "aggregate_expr": "host_resource_id"
-            }
-        ],
         "attribute_table": "account",
         "attribute_table_schema": "modw_cloud",
+        "attribute_filter_map_query": {
+            "account_id": "SELECT account_id FROM modw_cloud.account WHERE display in (__filter_values__)"
+        },
+        "attribute_description_query": "SELECT DISTINCT display AS filter_name FROM modw_cloud.account WHERE display IN (__filter_values__) ORDER BY display",
         "attribute_to_aggregate_table_key_map": [
             {
                 "account_id": "account_id"
@@ -139,10 +133,10 @@
                 "display"
             ],
             "records": {
-                "id": "account_id",
+                "id": "display",
                 "name": "display",
                 "order_id": "display",
-                "short_name": "provider_account"
+                "short_name": "display"
             }
         },
         "category": "Administrative",

From c559af1e693c97161e51aa5c890a12a599f2db10 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Tue, 17 Mar 2020 08:29:30 -0400
Subject: [PATCH 03/10] adding comments and removing unneeded changes

---
 tests/ci/bootstrap.sh                    | 7 ++++---
 tests/ci/scripts/xdmod-upgrade-cloud.tcl | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tests/ci/bootstrap.sh b/tests/ci/bootstrap.sh
index 79252d589b..ee61f165e4 100755
--- a/tests/ci/bootstrap.sh
+++ b/tests/ci/bootstrap.sh
@@ -72,7 +72,6 @@ then
     then
         sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
         sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
-        #sudo -u xdmod php /data/xdmod/tools/etl/etl_overseer.php -p ingest-cloud-resource-specs -d "CLOUD_RESOURCE_SPECS_DIRECTORY=$REF_DIR/openstack_resource_specs" -r openstack -v debug
     fi
     sudo -u xdmod xdmod-ingestor
 
@@ -95,6 +94,8 @@ fi
 if [ "$XDMOD_TEST_MODE" = "upgrade" ];
 then
     if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
+      # Ingesting data before running the yum install to upgrade xdmod allows us to
+      # more accurately test the case of upgrading when you have data for two cloud resources
       ~/bin/services start
       expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
       sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
@@ -128,8 +129,8 @@ then
 
     #
     if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
-        #expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
-
+        # Ingesting the previous data again allows us to test functionality that
+        # was added in the upgraded version such as the Provider and Domain group bys
         sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
         sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
         sudo -u xdmod xdmod-ingestor
diff --git a/tests/ci/scripts/xdmod-upgrade-cloud.tcl b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
index 8090d5ec13..78496a5625 100644
--- a/tests/ci/scripts/xdmod-upgrade-cloud.tcl
+++ b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
@@ -29,7 +29,7 @@ source [file join [file dirname [info script]] helper-functions.tcl]
 # they should typically not be changed as they need to match up with the
 # settings in the docker container
 
-set timeout 180
+set timeout 240
 spawn "xdmod-setup"
 
 #answerQuestion {Do you want to continue (yes, no)? [no] } Yes

From a205094679836f39e9aa4c345aa679c0c0c942d1 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Tue, 17 Mar 2020 08:47:05 -0400
Subject: [PATCH 04/10] adding documentation

---
 .../migrations/cloud-migration-8-5-1_9-0-0.sql         | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
index dfeae21ca6..e55ac42616 100644
--- a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
+++ b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
@@ -1,3 +1,9 @@
+-- In order to chnage from a multi column to single column unique auto increment id field
+-- we change in id field on the table to not be auto increment, then drop the auto increment
+-- key and finally drop the id field. Once ingestion is run again at the end of the migration
+-- pipeline the id field is remade as a unique field with auto incremented id's. The aggregation
+-- tables use the account_id and instance_type_id fields and are updated when aggregation is run
+-- at the end of the migration script
 ALTER TABLE modw_cloud.account MODIFY account_id INT NOT NULL;
 ALTER TABLE modw_cloud.account DROP INDEX autoincrement_key;
 ALTER TABLE modw_cloud.account DROP COLUMN account_id;
@@ -6,6 +12,10 @@ ALTER TABLE modw_cloud.instance_type MODIFY instance_type_id INT NOT NULL;
 ALTER TABLE modw_cloud.instance_type DROP INDEX increment_key;
 ALTER TABLE modw_cloud.instance_type DROP COLUMN instance_type_id;
 //
+-- The instance_id in used on the event table so we need to update the event table
+-- with the new instance_id value. The existing multi column auto increment key is
+-- dropped and a new field is created. Once the event table is updated the old instance_id
+-- is dropped and the new instance_id field is renamed
 ALTER TABLE modw_cloud.instance MODIFY instance_id INT(11) NOT NULL;
 ALTER TABLE modw_cloud.instance DROP INDEX increment_key;
 ALTER TABLE modw_cloud.instance ADD COLUMN new_instance_id INT(11) UNSIGNED NOT NULL auto_increment unique;

From 4e069d563f87bb40f3a5d960e620794fe4c4219c Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Tue, 17 Mar 2020 11:48:44 -0400
Subject: [PATCH 05/10] update test data

---
 tests/ci/bootstrap.sh                                  |  3 +++
 .../integration/lib/Controllers/UsageExplorerTest.php  | 10 +++++-----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/tests/ci/bootstrap.sh b/tests/ci/bootstrap.sh
index ee61f165e4..bcf8d90e10 100755
--- a/tests/ci/bootstrap.sh
+++ b/tests/ci/bootstrap.sh
@@ -102,6 +102,9 @@ then
       sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
       sudo -u xdmod xdmod-ingestor
       ~/bin/services stop
+      # Some temp files get made in /tmp and not deleted. This causes tests in validate.sh
+      # to file. Removing the files makes the test pass
+      find /tmp -type f -newer /usr/share/xdmod/html/index.php -delete
     fi
     yum -y install ~/rpmbuild/RPMS/*/*.rpm
     ~/bin/services start
diff --git a/tests/integration/lib/Controllers/UsageExplorerTest.php b/tests/integration/lib/Controllers/UsageExplorerTest.php
index 9f9272bbfb..a7bc0ad711 100644
--- a/tests/integration/lib/Controllers/UsageExplorerTest.php
+++ b/tests/integration/lib/Controllers/UsageExplorerTest.php
@@ -626,9 +626,9 @@ public function provideFilterIdLookup()
                     'realm' => 'Cloud',
                     'filters' => array(
                         array(
-                            array('project' => '3'),
-                            array('project_filter' => '3'),
-                            array('project_filter'=> '"zealous"')
+                            array('project' => 'zealous'),
+                            array('project_filter' => 'zealous'),
+                            array('project_filter'=> 'zealous')
                         )
                     ),
                     'expected' => array(
@@ -648,8 +648,8 @@ public function provideFilterIdLookup()
                     'realm' => 'Cloud',
                     'filters' => array(
                         array(
-                            array('project_filter' => '2,3,4'),
-                            array('project_filter'=> '\'zealous\',\'youthful\',\'zen\'')
+                            array('project_filter' => "zealous, youthful, zen"),
+                            array('project_filter'=> 'zealous, youthful, zen')
                         )
                     ),
                     'expected' => array(

From 06569c4394b3453da4e3f8b51464a749e55cf732 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Fri, 20 Mar 2020 14:29:36 -0400
Subject: [PATCH 06/10] updating upgrade procedure and adding chart filter test
 for instance type group by

---
 .../cloud-migration-8-5-1_9-0-0.sql           | 60 ++++++++++++++++++-
 .../xdmod/regression/chartFilterTests.json    | 15 +++++
 tests/ci/bootstrap.sh                         | 21 ++++---
 tests/ci/scripts/xdmod-upgrade-cloud.tcl      |  2 +-
 .../Controllers/MetricExplorerChartsTest.php  |  4 ++
 5 files changed, 87 insertions(+), 15 deletions(-)

diff --git a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
index e55ac42616..1c8aaf15d8 100644
--- a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
+++ b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
@@ -4,13 +4,47 @@
 -- pipeline the id field is remade as a unique field with auto incremented id's. The aggregation
 -- tables use the account_id and instance_type_id fields and are updated when aggregation is run
 -- at the end of the migration script
+//
+LOCK TABLES
+  modw_cloud.account WRITE,
+  modw_cloud.instance WRITE,
+  modw_cloud.instance_type WRITE,
+  modw_cloud.instance_data WRITE,
+  modw_cloud.session_records WRITE,
+  modw_cloud.event WRITE;
+//
 ALTER TABLE modw_cloud.account MODIFY account_id INT NOT NULL;
 ALTER TABLE modw_cloud.account DROP INDEX autoincrement_key;
-ALTER TABLE modw_cloud.account DROP COLUMN account_id;
+ALTER TABLE modw_cloud.account ADD COLUMN new_account_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.instance as i
+JOIN
+  modw_cloud.account as acc
+ON
+  i.resource_id = acc.resource_id AND i.account_id = acc.account_id
+SET
+  i.account_id = acc.new_account_id;
+
 //
 ALTER TABLE modw_cloud.instance_type MODIFY instance_type_id INT NOT NULL;
 ALTER TABLE modw_cloud.instance_type DROP INDEX increment_key;
-ALTER TABLE modw_cloud.instance_type DROP COLUMN instance_type_id;
+ALTER TABLE modw_cloud.instance_type ADD COLUMN new_instance_type_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.instance_data as id
+JOIN
+  modw_cloud.instance_type as it
+ON
+  id.resource_id = it.resource_id AND id.instance_type_id = it.instance_type_id
+SET
+  id.instance_type_id = it.new_instance_type_id;
+UPDATE
+  modw_cloud.session_records as sr
+JOIN
+  modw_cloud.instance_type as it
+ON
+  sr.resource_id = it.resource_id AND sr.instance_type_id = it.instance_type_id
+SET
+  sr.instance_type_id = it.new_instance_type_id;
 //
 -- The instance_id in used on the event table so we need to update the event table
 -- with the new instance_id value. The existing multi column auto increment key is
@@ -27,5 +61,25 @@ ON
   ev.resource_id = i.resource_id AND ev.instance_id = i.instance_id
 SET
   ev.instance_id = i.new_instance_id;
+-- On the session_records table the instance_id column is part of the primary key and
+-- in order to update it the primary key needs to be removed, update to the new instance_id
+-- and then reapply the primary key
+ALTER TABLE modw_cloud.session_records DROP PRIMARY KEY;
+UPDATE
+  modw_cloud.session_records as sr
+JOIN
+  modw_cloud.instance as i
+ON
+  sr.resource_id = i.resource_id AND sr.instance_id = i.instance_id
+SET
+  sr.instance_id = i.new_instance_id;
+ALTER TABLE modw_cloud.session_records ADD PRIMARY KEY (resource_id, instance_id, start_time_ts);
+//
+ALTER TABLE modw_cloud.account DROP COLUMN account_id;
+ALTER TABLE modw_cloud.account CHANGE new_account_id account_id INT(11) UNSIGNED;
+ALTER TABLE modw_cloud.instance_type DROP COLUMN instance_type_id;
+ALTER TABLE modw_cloud.instance_type CHANGE new_instance_type_id instance_type_id  INT(11) UNSIGNED;
 ALTER TABLE modw_cloud.instance DROP COLUMN instance_id;
-ALTER TABLE modw_cloud.instance CHANGE new_instance_id instance_id int(11) unsigned;
+ALTER TABLE modw_cloud.instance CHANGE new_instance_id instance_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+//
+UNLOCK TABLES;
diff --git a/tests/artifacts/xdmod/regression/chartFilterTests.json b/tests/artifacts/xdmod/regression/chartFilterTests.json
index cb3d6746e5..88116f6a8a 100644
--- a/tests/artifacts/xdmod/regression/chartFilterTests.json
+++ b/tests/artifacts/xdmod/regression/chartFilterTests.json
@@ -382,6 +382,21 @@
             "yvalue": 253.1089
         }
     },
+    {
+        "settings": {
+            "realm": "Cloud",
+            "metric": "cloud_core_time",
+            "date": "2019-06-26",
+            "filter_dimension": "configuration",
+            "filter_values": [
+                "m1.medium"
+            ]
+        },
+        "expected": {
+            "subtitle": "Instance Type =  m1.medium",
+            "yvalue": 29635.5067
+        }
+    },
     {
         "settings": {
             "realm": "Cloud",
diff --git a/tests/ci/bootstrap.sh b/tests/ci/bootstrap.sh
index bcf8d90e10..e58d0df4b2 100755
--- a/tests/ci/bootstrap.sh
+++ b/tests/ci/bootstrap.sh
@@ -93,19 +93,18 @@ fi
 
 if [ "$XDMOD_TEST_MODE" = "upgrade" ];
 then
-    if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
+    #if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
       # Ingesting data before running the yum install to upgrade xdmod allows us to
       # more accurately test the case of upgrading when you have data for two cloud resources
-      ~/bin/services start
-      expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
-      sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
-      sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
-      sudo -u xdmod xdmod-ingestor
-      ~/bin/services stop
+      #~/bin/services start
+      #expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
+      #sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
+      #sudo -u xdmod xdmod-ingestor
+      #~/bin/services stop
       # Some temp files get made in /tmp and not deleted. This causes tests in validate.sh
       # to file. Removing the files makes the test pass
-      find /tmp -type f -newer /usr/share/xdmod/html/index.php -delete
-    fi
+      #find /tmp -type f -newer /usr/share/xdmod/html/index.php -delete
+    #fi
     yum -y install ~/rpmbuild/RPMS/*/*.rpm
     ~/bin/services start
 
@@ -132,8 +131,8 @@ then
 
     #
     if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
-        # Ingesting the previous data again allows us to test functionality that
-        # was added in the upgraded version such as the Provider and Domain group bys
+        expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
+
         sudo -u xdmod xdmod-shredder -r openstack -d $REF_DIR/openstack -f openstack
         sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
         sudo -u xdmod xdmod-ingestor
diff --git a/tests/ci/scripts/xdmod-upgrade-cloud.tcl b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
index 78496a5625..37ce5b868d 100644
--- a/tests/ci/scripts/xdmod-upgrade-cloud.tcl
+++ b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
@@ -33,7 +33,7 @@ set timeout 240
 spawn "xdmod-setup"
 
 #answerQuestion {Do you want to continue (yes, no)? [no] } Yes
-confirmUpgrade
+#confirmUpgrade
 # Enter config settings for each resource
 selectMenuOption 4
 foreach resource $resources {
diff --git a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
index 03367c4577..42d1bf4619 100644
--- a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
+++ b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
@@ -70,6 +70,10 @@ public function testChartFilters($helper, $settings, $expected)
         $chartData = $chartStore->data[0];
 
         $this->assertEquals($expected['subtitle'], $chartData->subtitle->text);
+        if($expected['subtitle'] == "Instance Type =  c1.m1"){
+        //if($expected['subtitle'] == 'Domain = ( adjunct,  Default )'){
+          //var_dump($chartData);
+        }
         if (isset($expected['yvalue'])) {
             $this->assertEquals($expected['yvalue'], $chartData->series[0]->data[0]->y);
         } else {

From 7685f85ec839337806cb80ca13385b9b539f8f39 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Fri, 20 Mar 2020 15:25:33 -0400
Subject: [PATCH 07/10] update comments

---
 .../migrations/cloud-migration-8-5-1_9-0-0.sql       |  5 ++---
 tests/ci/bootstrap.sh                                | 12 ------------
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
index 1c8aaf15d8..a352dd31ed 100644
--- a/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
+++ b/configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql
@@ -24,7 +24,6 @@ ON
   i.resource_id = acc.resource_id AND i.account_id = acc.account_id
 SET
   i.account_id = acc.new_account_id;
-
 //
 ALTER TABLE modw_cloud.instance_type MODIFY instance_type_id INT NOT NULL;
 ALTER TABLE modw_cloud.instance_type DROP INDEX increment_key;
@@ -62,8 +61,8 @@ ON
 SET
   ev.instance_id = i.new_instance_id;
 -- On the session_records table the instance_id column is part of the primary key and
--- in order to update it the primary key needs to be removed, update to the new instance_id
--- and then reapply the primary key
+-- in order to update it the primary key needs to be removed then the instance_id can
+-- be updated and the primary key reapplied
 ALTER TABLE modw_cloud.session_records DROP PRIMARY KEY;
 UPDATE
   modw_cloud.session_records as sr
diff --git a/tests/ci/bootstrap.sh b/tests/ci/bootstrap.sh
index e58d0df4b2..e7786bdc9e 100755
--- a/tests/ci/bootstrap.sh
+++ b/tests/ci/bootstrap.sh
@@ -93,18 +93,6 @@ fi
 
 if [ "$XDMOD_TEST_MODE" = "upgrade" ];
 then
-    #if [[ "$XDMOD_REALMS" = *"cloud"* ]]; then
-      # Ingesting data before running the yum install to upgrade xdmod allows us to
-      # more accurately test the case of upgrading when you have data for two cloud resources
-      #~/bin/services start
-      #expect $BASEDIR/scripts/xdmod-upgrade-cloud.tcl | col -b
-      #sudo -u xdmod xdmod-shredder -r nutsetters -d $REF_DIR/nutsetters -f openstack
-      #sudo -u xdmod xdmod-ingestor
-      #~/bin/services stop
-      # Some temp files get made in /tmp and not deleted. This causes tests in validate.sh
-      # to file. Removing the files makes the test pass
-      #find /tmp -type f -newer /usr/share/xdmod/html/index.php -delete
-    #fi
     yum -y install ~/rpmbuild/RPMS/*/*.rpm
     ~/bin/services start
 

From ddccac8d9682207a0b000be74d64d0ae62e468d3 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Fri, 20 Mar 2020 15:45:44 -0400
Subject: [PATCH 08/10] removing debugging code

---
 .../regression/lib/Controllers/MetricExplorerChartsTest.php  | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
index 42d1bf4619..56f63cff68 100644
--- a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
+++ b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
@@ -70,10 +70,7 @@ public function testChartFilters($helper, $settings, $expected)
         $chartData = $chartStore->data[0];
 
         $this->assertEquals($expected['subtitle'], $chartData->subtitle->text);
-        if($expected['subtitle'] == "Instance Type =  c1.m1"){
-        //if($expected['subtitle'] == 'Domain = ( adjunct,  Default )'){
-          //var_dump($chartData);
-        }
+
         if (isset($expected['yvalue'])) {
             $this->assertEquals($expected['yvalue'], $chartData->series[0]->data[0]->y);
         } else {

From b7b59f42f0771b7f36682f62704b6d8621e5babb Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Fri, 20 Mar 2020 15:51:15 -0400
Subject: [PATCH 09/10] removing unnecessary changes

---
 tests/regression/lib/Controllers/MetricExplorerChartsTest.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
index 56f63cff68..03367c4577 100644
--- a/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
+++ b/tests/regression/lib/Controllers/MetricExplorerChartsTest.php
@@ -70,7 +70,6 @@ public function testChartFilters($helper, $settings, $expected)
         $chartData = $chartStore->data[0];
 
         $this->assertEquals($expected['subtitle'], $chartData->subtitle->text);
-
         if (isset($expected['yvalue'])) {
             $this->assertEquals($expected['yvalue'], $chartData->series[0]->data[0]->y);
         } else {

From 0266912452b77ae47cfd47fe9880224634537fb7 Mon Sep 17 00:00:00 2001
From: Greg Dean <gmdean@buffalo.edu>
Date: Fri, 20 Mar 2020 15:55:14 -0400
Subject: [PATCH 10/10] more removal of unneeded changes

---
 tests/ci/scripts/xdmod-upgrade-cloud.tcl | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/tests/ci/scripts/xdmod-upgrade-cloud.tcl b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
index 37ce5b868d..74c0242f72 100644
--- a/tests/ci/scripts/xdmod-upgrade-cloud.tcl
+++ b/tests/ci/scripts/xdmod-upgrade-cloud.tcl
@@ -3,14 +3,6 @@
 # XDMoD instance. This script will fail if run against an already installed
 # XDMoD.
 
-proc confirmUpgrade { } {
-    expect {
-        timeout { send_user "\nFailed to get prompt\n"; exit 1 }
-        -re "\nDo you want to continue .*\\\] "
-    }
-    send yes\n
-}
-
 #-------------------------------------------------------------------------------
 # Configuration settings for the XDMoD resources
 
@@ -32,8 +24,6 @@ source [file join [file dirname [info script]] helper-functions.tcl]
 set timeout 240
 spawn "xdmod-setup"
 
-#answerQuestion {Do you want to continue (yes, no)? [no] } Yes
-#confirmUpgrade
 # Enter config settings for each resource
 selectMenuOption 4
 foreach resource $resources {