Add PI information for the Cloud realm

bin/xdmod-import-csv

@@ -216,6 +216,9 @@ function main()
             case 'group-to-hierarchy':
                 updateGroupToHierarchy($data);
                 break;
+            case 'cloud-project-to-pi':
+                updateCloudProjectToPI($data);
+                break;
             default:
                 $logger->crit("Unknown type '$type'");
                 exit(1);
@@ -243,6 +246,22 @@ function main()
     exit;
 }
 
+function updateCloudProjectToPI(array $piToProject){
+    global $db, $logger;
+    $logger->debug('Updating PI information for Cloud realm');
+    $sql = "
+    INSERT INTO
+      modw_cloud.staging_pi_to_project (pi_name, project_name, resource_name)
+    VALUES
+      (:pi, :project_name, :resource_name) ON DUPLICATE KEY UPDATE pi_name = VALUES(pi_name), resource_name = VALUES(resource_name)";
+    foreach($piToProject as $row){
+        $db->execute($sql, array(
+          ":pi" => $row[0],
+          ":project_name" => $row[1],
+          ":resource_name" => $row[2]
+        ));
+    }
+}
 /**
  * Update people's names.
  *
@@ -769,6 +788,10 @@ row.
 
         Fields: groupname, hierachy item name
 
+    cloud-project-to-pi
+
+        Fields: pi_name, project_name, resource
+
 NOTE: After updating any names or hierarchy items it is necessary to
 reingest your data for the changes to be applied to the datawarehouse.
 Hierarchy changes also require reaggregating data for the entire time

classes/OpenXdmod/DataWarehouseInitializer.php

@@ -235,7 +235,15 @@ public function ingestCloudDataOpenStack()
         try {
             $this->logger->notice('Ingesting OpenStack event log data');
             Utilities::runEtlPipeline(
-                array('jobs-cloud-common', 'jobs-cloud-import-users-openstack', 'jobs-cloud-extract-openstack'),
+                array(
+                  'staging-ingest-common',
+                  'jobs-cloud-common',
+                  'jobs-cloud-import-users-openstack',
+                  'jobs-cloud-ingest-pi',
+                  'hpcdb-ingest-common',
+                  'hpcdb-xdw-ingest-common',
+                  'jobs-cloud-extract-openstack'
+                ),
                 $this->logger
             );
         }

configuration/datawarehouse.d/ref/Cloud-group-bys.json

@@ -307,5 +307,8 @@
     },
     "year": {
         "$ref": "datawarehouse.d/ref/group-by-time-period.json#/year"
+    },
+    "pi": {
+        "$ref": "datawarehouse.d/ref/group-by-common.json#/pi"
     }
 }

configuration/etl/etl.d/jobs_cloud_common.json

@@ -25,6 +25,13 @@
             "options_class": "IngestorOptions",
             "truncate_destination": false,
             "enabled": true
+        },
+        "jobs-cloud-ingest-pi": {
+            "namespace": "ETL\\Ingestor",
+            "options_class": "IngestorOptions",
+            "class": "DatabaseIngestor",
+            "truncate_destination": false,
+            "enabled": true
         }
     },
 
@@ -53,7 +60,8 @@
                 "cloud_generic/raw_volume.json",
                 "cloud_common/domains.json",
                 "cloud_openstack/domain_submission_venue_staging.json",
-                "cloud_openstack/domain_submission_venue.json"
+                "cloud_openstack/domain_submission_venue.json",
+                "cloud_common/staging_cloud_project_to_pi.json"
             ]
         },
         {
@@ -180,5 +188,72 @@
                 }
             }
         }
+    ],
+    "jobs-cloud-ingest-pi": [
+        {
+            "name": "pi-resource",
+            "description": "Ingest storage PI/Resource permutations",
+            "definition_file": "cloud_common/pi-resource.json",
+            "endpoints": {
+                "destination": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "pi",
+            "description": "Ingest storage PIs",
+            "definition_file": "cloud_common/pi.json",
+            "endpoints": {
+                "destination": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "union-user-pi--pi",
+            "description": "Ingest storage PIs (combined with users)",
+            "definition_file": "cloud_common/union-user-pi--pi.json",
+            "endpoints": {
+                "destination": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "union-user-pi-resource--pi",
+            "description": "Ingest storage PI/Resource permutations (combined with User/Resource permutations)",
+            "definition_file": "cloud_common/union-user-pi-resource--pi.json",
+            "endpoints": {
+                "destination": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "mod_shredder"
+                }
+            }
+        },
+        {
+            "name": "user-pi-resource",
+            "description": "Ingest storage User/PI/Resource permutations",
+            "definition_file": "cloud_common/user-pi-resource.json",
+            "endpoints": {
+                "destination": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "mod_shredder"
+                }
+            }
+        }
     ]
 }

configuration/etl/etl.d/jobs_cloud_generic.json

@@ -245,7 +245,15 @@
             "name": "GenericCloudAccountIngestor",
             "description": "Generic cloud account data",
             "class": "DatabaseIngestor",
-            "definition_file": "cloud_generic/account.json"
+            "definition_file": "cloud_generic/account.json",
+            "endpoints": {
+                "utility": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "modw"
+                }
+            }
         },
         {
             "name": "GenericCloudHostIngestor",

configuration/etl/etl.d/jobs_cloud_openstack.json

@@ -220,7 +220,15 @@
             "name": "OpenStackCloudAccountIngestor",
             "description": "OpenStack cloud account data",
             "class": "DatabaseIngestor",
-            "definition_file": "cloud_openstack/account.json"
+            "definition_file": "cloud_openstack/account.json",
+            "endpoints": {
+                "utility": {
+                    "type" : "mysql",
+                    "name" : "Cloud DB",
+                    "config" : "datawarehouse",
+                    "schema": "modw"
+                }
+            }
         },
         {
             "name": "OpenStackCloudHostIngestor",

configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json

@@ -45,7 +45,8 @@
                 "wallduration": "COALESCE(SUM(${wallduration_case_statement}), 0)",
                 "submission_venue_id": "sr.submission_venue_id",
                 "domain_id": "sr.domain_id",
-                "service_provider": "sr.service_provider"
+                "service_provider": "sr.service_provider",
+                "principalinvestigator_person_id": "a.principalinvestigator_person_id"
             },
             "groupby": [
               "${AGGREGATION_UNIT}_id",
@@ -74,6 +75,11 @@
                   "schema": "${UTILITY_SCHEMA}",
                   "alias": "task_resource",
                   "on": "task_resource.id = sr.resource_id"
+              },{
+                  "name": "account",
+                  "schema": "${SOURCE_SCHEMA}",
+                  "alias": "a",
+                  "on": "instance.account_id = a.account_id"
               }
             ],
             "where": [

configuration/etl/etl_action_defs.d/cloud_common/pi-resource.json

@@ -0,0 +1,18 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/pi-resource.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "pi_name": "DISTINCT pi.pi_name",
+            "resource_name": "pi.resource_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_common/pi.json

@@ -0,0 +1,17 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/pi.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "pi_name": "DISTINCT pi.pi_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_common/union-user-pi--pi.json

@@ -0,0 +1,17 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/union-user-pi.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "union_user_pi_name": "DISTINCT pi.pi_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_common/union-user-pi--user.json

@@ -0,0 +1,17 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/union-user-pi.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "union_user_pi_name": "DISTINCT pi.user_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_common/union-user-pi-resource--pi.json

@@ -0,0 +1,18 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/union-user-pi-resource.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "union_user_pi_name": "DISTINCT pi.pi_name",
+            "resource_name": "pi.resource_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_common/user-pi-resource.json

@@ -0,0 +1,19 @@
+{
+    "table_definition": {
+        "$ref": "${table_definition_dir}/common/staging/user-pi-resource.json#/table_definition"
+    },
+    "source_query": {
+        "records": {
+            "user_name": "DISTINCT pi.pi_name",
+            "pi_name": "pi.pi_name",
+            "resource_name": "pi.resource_name"
+        },
+        "joins": [
+            {
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_pi_to_project",
+                "alias": "pi"
+            }
+        ]
+    }
+}

configuration/etl/etl_action_defs.d/cloud_generic/account.json

@@ -12,14 +12,43 @@
         "records": {
             "resource_id": "raw.resource_id",
             "provider_account": "raw.provider_account",
-            "display": "raw.provider_account"
+            "display": "raw.provider_account",
+            "principalinvestigator_person_id": "COALESCE(p.person_id, -1)"
         },
 
         "joins": [
             {
                 "name": "generic_cloud_raw_event",
                 "schema": "${SOURCE_SCHEMA}",
                 "alias": "raw"
+            },
+            {
+                "name": "resourcefact",
+                "schema": "${UTILITY_SCHEMA}",
+                "alias": "rf",
+                "type": "LEFT",
+                "on": "raw.resource_id = rf.id"
+            },
+            {
+                "name": "staging_pi_to_project",
+                "schema": "${SOURCE_SCHEMA}",
+                "alias": "pi",
+                "type": "LEFT",
+                "on": "raw.provider_account = pi.project_name AND rf.code = pi.resource_name"
+            },
+            {
+                "name": "account",
+                "schema": "${UTILITY_SCHEMA}",
+                "alias": "a",
+                "type": "LEFT",
+                "on": "pi.pi_name = a.charge_number"
+            },
+            {
+                "name": "principalinvestigator",
+                "schema": "${UTILITY_SCHEMA}",
+                "alias": "p",
+                "type": "LEFT",
+                "on": "a.id = p.request_id"
             }
         ],