Add extra column to record the 'real' application name for proprietar…

…y applications
ubccr · Jan 19, 2023 · f13d7bc · f13d7bc
1 parent f6753df
commit f13d7bc
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 5 deletions.
diff --git a/configuration/etl/etl.d/supremm-migration-10_0_0-10_5_0.json b/configuration/etl/etl.d/supremm-migration-10_0_0-10_5_0.json
@@ -13,5 +13,28 @@
         }
     },
     "migration-10_0_0-10_5_0": [
+        {
+            "name": "table-create",
+            "description": "Setup tables",
+            "class": "ManageTables",
+            "namespace": "ETL\\Maintenance",
+            "options_class": "MaintenanceOptions",
+            "definition_file_list": [
+                "supremm/job.json",
+                "supremm/job_errors.json"
+            ]
+        },
+        {
+            "name": "update-table-contents",
+            "description": "Update non-etlv2-managed tables and contents",
+            "namespace": "ETL\\Maintenance",
+            "options_class": "MaintenanceOptions",
+            "class": "ExecuteSql",
+            "sql_file_list": [
+                {
+                    "sql_file": "supremm/migration-10_0_0-10_5_0.sql"
+                }
+            ]
+        }
     ]
 }
diff --git a/configuration/etl/etl_sql.d/supremm/migration-10_0_0-10_5_0.sql b/configuration/etl/etl_sql.d/supremm/migration-10_0_0-10_5_0.sql
@@ -0,0 +1,2 @@
+ALTER TABLE `modw_supremm`.`executable` ADD COLUMN `real_application_id` int(11) DEFAULT '-1' AFTER `application_id`
+//
diff --git a/etl/js/config/supremm/dataset_maps/pcp.js b/etl/js/config/supremm/dataset_maps/pcp.js
@@ -32,13 +32,15 @@ module.exports = function(config) {
                 if (job.procDump.constrained.length > 0) {
                     return {
                         executable: job.procDump.constrained[0],
-                        name: 'uncategorized'
+                        name: 'uncategorized',
+                        realname: 'uncategorized'
                     };
                 }
                 if (job.procDump.unconstrained.length > 0) {
                     return {
                         executable: job.procDump.unconstrained[0],
-                        name: 'uncategorized'
+                        name: 'uncategorized',
+                        realname: 'uncategorized'
                     };
                 }
             }
@@ -195,6 +197,21 @@ module.exports = function(config) {
                     };
                 }
             },
+            real_application: {
+                formula: function (job) {
+                    var app = getProcInfo(job);
+                    if (app) {
+                        return {
+                            value: app.realname,
+                            error: 0
+                        };
+                    }
+                    return {
+                        value: null,
+                        error: this.metricErrors.codes.metricMissingUnknownReason.value
+                    };
+                }
+            },
             "exit_status": {
                 formula: function(job) {
                     var exit = this.ref(job, "acct.exit_status");

diff --git a/etl/js/config/supremm/etl.schema.js b/etl/js/config/supremm/etl.schema.js
@@ -327,9 +327,14 @@ module.exports = {
             per: "job",
             dim_insert: function(attributes) {
                 return {
-                    query: "INSERT IGNORE INTO modw_supremm.executable (`resource_id`, `exec`, `binary`, `exec_md5`, `application_id`) " +
-                           "VALUES (:resource_id, :exec, substring_index(:exec,'/',-1), MD5(:exec), COALESCE((SELECT id FROM modw_supremm.application WHERE `name` = :appname), -1) )",
-                    values: {resource_id: attributes.resource_id.value, exec: attributes.executable.value, appname: attributes.application.value}
+                    query: 'INSERT IGNORE INTO modw_supremm.executable (`resource_id`, `exec`, `binary`, `exec_md5`, `application_id`, `real_application_id`) ' +
+                           'VALUES (:resource_id, :exec, substring_index(:exec,\'/\',-1), MD5(:exec), COALESCE((SELECT id FROM modw_supremm.application WHERE `name` = :appname), -1), COALESCE((SELECT id FROM modw_supremm.application WHERE `name` = :real_appname), -1) )',
+                    values: {
+                        resource_id: attributes.resource_id.value,
+                        exec: attributes.executable.value,
+                        appname: attributes.application.value,
+                        real_appname: attributes.real_application.value
+                    }
                 };
             }
         },
@@ -343,6 +348,16 @@ module.exports = {
             comments: "The name of the application that ran",
             per: "job",
         },
+        real_application: {
+            unit: null,
+            type: 'string',
+            dtype: 'accounting',
+            group: 'Executable',
+            nullable: false,
+            def: 'NA',
+            comments: 'The actual name of the application that ran.',
+            per: 'job'
+        },
         exit_status: {
             unit: null,
             type: "string",
@@ -2335,6 +2350,11 @@ module.exports = {
             table: "modw_supremm.application",
             where: "name = :application"
         },
+        real_application: {
+            mapping: { real_application_id: 'id' },
+            table: 'modw_supremm.application',
+            where: 'name = :real_application'
+        },
         cwd: {
             mapping: {cwd_id: "id"},
             table: "modw_supremm.cwd",
@@ -2599,6 +2619,20 @@ module.exports = {
                 table: 'application'
             }
         },
+        real_application_id: {
+            type: 'int32',
+            nullable: false,
+            def: null,
+            group: 'Executable',
+            comments: 'The actual application.',
+            per: 'job',
+            table: 'job',
+            queries: ['real_application'],
+            agg: [{
+                table: 'supremmfact',
+                dimension: true
+            }]
+        },
         nodecount_id: {
             type: "int32",
             nullable: false,

diff --git a/etl/js/config/supremm/output_db/modw_supremm.sql b/etl/js/config/supremm/output_db/modw_supremm.sql
@@ -306,6 +306,7 @@ CREATE TABLE `executable` (
   `exec_md5` char(32) COLLATE utf8_unicode_ci NOT NULL DEFAULT '',
   `binary` varchar(45) COLLATE utf8_unicode_ci DEFAULT NULL,
   `application_id` int(11) DEFAULT '-1',
+  `real_application_id` int(11) DEFAULT '-1',
   PRIMARY KEY (`id`),
   UNIQUE KEY `unique` (`exec_md5`,`resource_id`),
   KEY `exec_md5` (`exec_md5`,`resource_id`,`id`)

diff --git a/tests/integration_tests/scripts/validate.sh b/tests/integration_tests/scripts/validate.sh
@@ -51,6 +51,7 @@ checkForColumn netdir_util_write
 checkForColumn gpus
 checkForColumn gpu_time
 checkForColumn gpu_usage
+checkForColumn real_application_id
 
 # Check that the catastrophe buckets table has the extra column
 if [ -z "$(echo "show columns from modw_supremm.catastrophe_buckets LIKE 'h_description'" | mysql -N modw_supremm)" ];
@@ -59,6 +60,13 @@ then
     exitcode=1
 fi
 
+# Check that the executable table has the extra column
+if [ -z "$(echo "show columns from modw_supremm.executable LIKE 'real_application_id'" | mysql -N modw_supremm)" ];
+then
+    echo "Missing real_application_id column from modw_supremm.executable table"
+    exitcode=1
+fi
+
 # Check that the jobhosts table has end_time_ts column with non-zero timestamps
 jobcount=$(echo 'SELECT COUNT(*) FROM modw_supremm.job j, modw_supremm.jobhost jh WHERE j.resource_id = jh.resource_id AND j.local_job_id = jh.local_job_id AND j.end_time_ts = jh.end_time_ts' | mysql -N modw_supremm)
 if [ $jobcount -eq 0 ];