Merge pull request #1263 from eiffel777/fix-unique-columns-cloud-tables

Change account, instance and instance_type tables to have single unique auto increment column

configuration/datawarehouse.d/ref/Cloud-group-bys.json

@@ -2,13 +2,6 @@
     "configuration": {
         "name": "Instance Type",
         "description_html": "The instance type of a virtual machine.",
-        "additional_join_constraints": [
-            {
-                "attribute_expr": "resource_id",
-                "operation": "=",
-                "aggregate_expr": "host_resource_id"
-            }
-        ],
         "attribute_table_schema": "modw_cloud",
         "attribute_table": "instance_type",
         "attribute_to_aggregate_table_key_map": [
@@ -19,10 +12,10 @@
         "order": 6,
         "attribute_values_query": {
             "records": {
-                "id": "instance_type_id",
-                "short_name": "instance_type",
+                "id": "display",
+                "short_name": "display",
                 "name": "display",
-                "order_id": "instance_type_id"
+                "order_id": "display"
             },
             "joins": [
                 {
@@ -33,7 +26,11 @@
                 "instance_type_id"
             ],
             "query_hint": "DISTINCT"
-        }
+        },
+        "attribute_filter_map_query": {
+            "instance_type_id": "SELECT instance_type_id FROM modw_cloud.instance_type WHERE display IN (__filter_values__)"
+        },
+        "attribute_description_query": "SELECT DISTINCT display AS filter_name FROM modw_cloud.instance_type WHERE display IN (__filter_values__) ORDER BY display"
     },
     "domain": {
         "alternate_group_by_columns": [
@@ -115,15 +112,12 @@
         "name": "User"
     },
     "project": {
-        "additional_join_constraints": [
-            {
-                "attribute_expr": "resource_id",
-                "operation": "=",
-                "aggregate_expr": "host_resource_id"
-            }
-        ],
         "attribute_table": "account",
         "attribute_table_schema": "modw_cloud",
+        "attribute_filter_map_query": {
+            "account_id": "SELECT account_id FROM modw_cloud.account WHERE display in (__filter_values__)"
+        },
+        "attribute_description_query": "SELECT DISTINCT display AS filter_name FROM modw_cloud.account WHERE display IN (__filter_values__) ORDER BY display",
         "attribute_to_aggregate_table_key_map": [
             {
                 "account_id": "account_id"
@@ -139,10 +133,10 @@
                 "display"
             ],
             "records": {
-                "id": "account_id",
+                "id": "display",
                 "name": "display",
                 "order_id": "display",
-                "short_name": "provider_account"
+                "short_name": "display"
             }
         },
         "category": "Administrative",

configuration/etl/etl.d/xdmod-migration-8_5_1-9_0_0.json

@@ -1,6 +1,24 @@
 {
     "module": "xdmod",
     "migration-8_5_1-9_0_0": [
+        {
+            "name": "DeleteCloudAutoincrementColumns",
+            "description": "Post-ingest updates for massaging Open Stack data",
+            "namespace": "ETL\\Maintenance",
+            "options_class": "MaintenanceOptions",
+            "class": "ExecuteSql",
+            "sql_file_list": [
+                "migrations/cloud-migration-8-5-1_9-0-0.sql"
+            ],
+            "endpoints": {
+                "destination": {
+                    "type": "mysql",
+                    "name": "Cloud Realm Tables",
+                    "config": "database",
+                    "schema": "modw_cloud"
+                }
+            }
+        },
         {
             "name": "update-modw_cloud-tables",
             "description": "Update modw_cloud tables",
@@ -15,6 +33,8 @@
                 "cloud_openstack/raw_event.json",
                 "cloud_common/session_records.json",
                 "cloud_common/event.json",
+                "cloud_common/account.json",
+                "cloud_common/instance_type.json",
                 "cloud_common/instance.json"
             ],
             "endpoints": {

configuration/etl/etl_sql.d/migrations/cloud-migration-8-5-1_9-0-0.sql

@@ -0,0 +1,84 @@
+-- In order to chnage from a multi column to single column unique auto increment id field
+-- we change in id field on the table to not be auto increment, then drop the auto increment
+-- key and finally drop the id field. Once ingestion is run again at the end of the migration
+-- pipeline the id field is remade as a unique field with auto incremented id's. The aggregation
+-- tables use the account_id and instance_type_id fields and are updated when aggregation is run
+-- at the end of the migration script
+//
+LOCK TABLES
+  modw_cloud.account WRITE,
+  modw_cloud.instance WRITE,
+  modw_cloud.instance_type WRITE,
+  modw_cloud.instance_data WRITE,
+  modw_cloud.session_records WRITE,
+  modw_cloud.event WRITE;
+//
+ALTER TABLE modw_cloud.account MODIFY account_id INT NOT NULL;
+ALTER TABLE modw_cloud.account DROP INDEX autoincrement_key;
+ALTER TABLE modw_cloud.account ADD COLUMN new_account_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.instance as i
+JOIN
+  modw_cloud.account as acc
+ON
+  i.resource_id = acc.resource_id AND i.account_id = acc.account_id
+SET
+  i.account_id = acc.new_account_id;
+//
+ALTER TABLE modw_cloud.instance_type MODIFY instance_type_id INT NOT NULL;
+ALTER TABLE modw_cloud.instance_type DROP INDEX increment_key;
+ALTER TABLE modw_cloud.instance_type ADD COLUMN new_instance_type_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.instance_data as id
+JOIN
+  modw_cloud.instance_type as it
+ON
+  id.resource_id = it.resource_id AND id.instance_type_id = it.instance_type_id
+SET
+  id.instance_type_id = it.new_instance_type_id;
+UPDATE
+  modw_cloud.session_records as sr
+JOIN
+  modw_cloud.instance_type as it
+ON
+  sr.resource_id = it.resource_id AND sr.instance_type_id = it.instance_type_id
+SET
+  sr.instance_type_id = it.new_instance_type_id;
+//
+-- The instance_id in used on the event table so we need to update the event table
+-- with the new instance_id value. The existing multi column auto increment key is
+-- dropped and a new field is created. Once the event table is updated the old instance_id
+-- is dropped and the new instance_id field is renamed
+ALTER TABLE modw_cloud.instance MODIFY instance_id INT(11) NOT NULL;
+ALTER TABLE modw_cloud.instance DROP INDEX increment_key;
+ALTER TABLE modw_cloud.instance ADD COLUMN new_instance_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+UPDATE
+  modw_cloud.event AS ev
+JOIN
+  modw_cloud.instance AS i
+ON
+  ev.resource_id = i.resource_id AND ev.instance_id = i.instance_id
+SET
+  ev.instance_id = i.new_instance_id;
+-- On the session_records table the instance_id column is part of the primary key and
+-- in order to update it the primary key needs to be removed then the instance_id can
+-- be updated and the primary key reapplied
+ALTER TABLE modw_cloud.session_records DROP PRIMARY KEY;
+UPDATE
+  modw_cloud.session_records as sr
+JOIN
+  modw_cloud.instance as i
+ON
+  sr.resource_id = i.resource_id AND sr.instance_id = i.instance_id
+SET
+  sr.instance_id = i.new_instance_id;
+ALTER TABLE modw_cloud.session_records ADD PRIMARY KEY (resource_id, instance_id, start_time_ts);
+//
+ALTER TABLE modw_cloud.account DROP COLUMN account_id;
+ALTER TABLE modw_cloud.account CHANGE new_account_id account_id INT(11) UNSIGNED;
+ALTER TABLE modw_cloud.instance_type DROP COLUMN instance_type_id;
+ALTER TABLE modw_cloud.instance_type CHANGE new_instance_type_id instance_type_id  INT(11) UNSIGNED;
+ALTER TABLE modw_cloud.instance DROP COLUMN instance_id;
+ALTER TABLE modw_cloud.instance CHANGE new_instance_id instance_id INT(11) UNSIGNED NOT NULL auto_increment unique;
+//
+UNLOCK TABLES;

configuration/etl/etl_tables.d/cloud_common/account.json

@@ -55,7 +55,6 @@
             {
                 "name": "autoincrement_key",
                 "columns": [
-                    "resource_id",
                     "account_id"
                 ],
                 "is_unique": true

configuration/etl/etl_tables.d/cloud_common/instance.json

@@ -62,15 +62,8 @@
                 "is_unique": true
             },
             {
-                "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a",
-                "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column",
-                "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is",
-                "#": "useful when you want to put data into ordered groups.",
-                "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)",
-                "#": "https://www.ryadel.com/en/mysql-two-columns-primary-key-with-auto-increment/",
                 "name": "increment_key",
                 "columns": [
-                    "resource_id",
                     "instance_id"
                 ],
                 "is_unique": true

configuration/etl/etl_tables.d/cloud_common/instance_type.json

@@ -89,15 +89,8 @@
                 "is_unique": true
             },
             {
-                "#": "For MyISAM tables, you can specify AUTO_INCREMENT on a secondary column in a",
-                "#": "multiple-column index. In this case, the generated value for the AUTO_INCREMENT column",
-                "#": "is calculated as MAX(auto_increment_column) + 1 WHERE prefix=given-prefix. This is",
-                "#": "useful when you want to put data into ordered groups.",
-                "#": "See [MyISAM Notes](https://dev.mysql.com/doc/refman/5.7/en/example-auto-increment.html)",
-
                 "name": "increment_key",
                 "columns": [
-                    "resource_id",
                     "instance_type_id"
                 ],
                 "is_unique": true

tests/artifacts/xdmod/regression/chartFilterTests.json

@@ -382,6 +382,21 @@
             "yvalue": 253.1089
         }
     },
+    {
+        "settings": {
+            "realm": "Cloud",
+            "metric": "cloud_core_time",
+            "date": "2019-06-26",
+            "filter_dimension": "configuration",
+            "filter_values": [
+                "m1.medium"
+            ]
+        },
+        "expected": {
+            "subtitle": "Instance Type =  m1.medium",
+            "yvalue": 29635.5067
+        }
+    },
     {
         "settings": {
             "realm": "Cloud",

tests/integration/lib/Controllers/UsageExplorerTest.php

@@ -626,9 +626,9 @@ public function provideFilterIdLookup()
                     'realm' => 'Cloud',
                     'filters' => array(
                         array(
-                            array('project' => '3'),
-                            array('project_filter' => '3'),
-                            array('project_filter'=> '"zealous"')
+                            array('project' => 'zealous'),
+                            array('project_filter' => 'zealous'),
+                            array('project_filter'=> 'zealous')
                         )
                     ),
                     'expected' => array(
@@ -648,8 +648,8 @@ public function provideFilterIdLookup()
                     'realm' => 'Cloud',
                     'filters' => array(
                         array(
-                            array('project_filter' => '2,3,4'),
-                            array('project_filter'=> '\'zealous\',\'youthful\',\'zen\'')
+                            array('project_filter' => "zealous, youthful, zen"),
+                            array('project_filter'=> 'zealous, youthful, zen')
                         )
                     ),
                     'expected' => array(