Add GPU tracking to Jobs realm

classes/OpenXdmod/Migration/Version851To900/DatabaseMigration.php

@@ -5,12 +5,18 @@
 use OpenXdmod\Setup\Console;
 use FilterListBuilder;
 use CCR\DB;
+use ETL\Utilities;
+use Xdmod\SlurmGresParser;
 
 /**
 * Migrate databases from version 8.5.1 to 9.0.0.
 */
 class DatabasesMigration extends \OpenXdmod\Migration\DatabasesMigration
 {
+    /**
+     * Update batch export request realm names, rebuild storage filters lists.
+     * Prompt user and re-ingest slurm GPU data if desired.
+     */
     public function execute()
     {
         parent::execute();
@@ -29,5 +35,63 @@ public function execute()
             $this->logger->warning('Failed to build filter list: '  . $e->getMessage());
             $this->logger->warning('You may need to run xdmod-build-filter-lists manually');
         }
+
+        $console = Console::factory();
+        $console->displayMessage(<<<"EOT"
+This version of Open XDMoD has support for GPU metrics in the jobs realm.  If
+you have shredded Slurm job records in the past it is possible to extract the
+GPU count from the ReqGRES data that was collected.  This will require
+re-ingest and re-aggregating your job data.
+EOT
+        );
+        $console->displayBlankLine();
+        $reingestSlurmData = $console->prompt(
+            'Re-ingest and re-aggregate Slurm job data?',
+            'yes',
+            ['yes', 'no']
+        );
+
+        if ($reingestSlurmData === 'yes') {
+            Utilities::runEtlPipeline(['jobs-gpu-migration-8_5_1-9_0_0'], $this->logger);
+            $this->updateSlurmGpuCount();
+            // Use current time from the database in case clocks are not
+            // synchronized.
+            $lastModifiedStartDate = DB::factory('hpcdb')->query('SELECT NOW() AS now FROM dual')[0]['now'];
+            Utilities::runEtlPipeline(
+                ['jobs-gpu-re-ingest-8_5_1-9_0_0', 'jobs-xdw-aggregate'],
+                $this->logger,
+                ['last-modified-start-date' => $lastModifiedStartDate]
+            );
+            $builder = new FilterListBuilder();
+            $builder->setLogger($this->logger);
+            $builder->buildRealmLists('Jobs');
+        }
+    }
+
+    /**
+     * Update the GPU count for all slurm records in
+     * `mod_shredder`.`shredded_job_slurm` that have ReqGRES data.
+     */
+    private function updateSlurmGpuCount()
+    {
+        $dbh = DB::factory('shredder');
+        $dbh->beginTransaction();
+        try {
+            $this->logger->notice('Querying slurm job records');
+            $rows = $dbh->query("SELECT shredded_job_slurm_id AS id, req_gres FROM shredded_job_slurm WHERE req_gres != ''");
+            $this->logger->notice('Updating slurm job records');
+            $sth = $dbh->prepare('UPDATE shredded_job_slurm SET ngpus = :gpuCount WHERE shredded_job_slurm_id = :id');
+            $gresParser = new SlurmGresParser();
+            foreach ($rows as $row) {
+                $gres = $gresParser->parseReqGres($row['req_gres']);
+                $gpuCount = $gresParser->getGpuCountFromGres($gres);
+                $sth->execute(['gpuCount' => $gpuCount, 'id' => $row['id']]);
+            }
+            $dbh->commit();
+            $this->logger->notice('Done updating slurm job records');
+        } catch (Exception $e) {
+            $dbh->rollBack();
+            $this->logger->err('Failed to update slurm job records: '  . $e->getMessage());
+        }
     }
 }

classes/OpenXdmod/Shredder/Slurm.php

@@ -12,6 +12,7 @@
 use DateTimeZone;
 use CCR\DB\iDatabase;
 use OpenXdmod\Shredder;
+use Xdmod\SlurmGresParser;
 
 class Slurm extends Shredder
 {
@@ -126,6 +127,7 @@ class Slurm extends Shredder
         'wait_time'       => 'GREATEST(CAST(start_time AS SIGNED) - CAST(submit_time AS SIGNED), 0)',
         'node_count'      => 'nnodes',
         'cpu_count'       => 'ncpus',
+        'gpu_count'       => 'ngpus',
         'cpu_req'         => 'req_cpus',
         'mem_req'         => 'req_mem',
         'timelimit'       => 'timelimit',
@@ -170,6 +172,11 @@ class Slurm extends Shredder
      */
     protected $timeZone;
 
+    /**
+     * @var \Xdmod\SlurmGresParser
+     */
+    private $gresParser;
+
     /**
      * @inheritdoc
      */
@@ -180,6 +187,7 @@ public function __construct(iDatabase $db)
         self::$columnCount = count(self::$columnNames);
 
         $this->timeZone = new DateTimeZone('UTC');
+        $this->gresParser = new SlurmGresParser();
     }
 
     /**
@@ -251,6 +259,9 @@ public function shredLine($line)
             $job[$key] = $this->parseTimeField($job[$key]);
         }
 
+        $gres = $this->gresParser->parseReqGres($job['req_gres']);
+        $job['ngpus'] = $this->gresParser->getGpuCountFromGres($gres);
+
         $job['cluster_name'] = $this->getResource();
 
         $this->checkJobData($line, $job);

classes/Xdmod/SlurmGresParser.php

@@ -0,0 +1,69 @@
+<?php
+/**
+ * Slurm GRES parser.
+ */
+
+namespace Xdmod;
+
+/**
+ * Contains functions related to Slurm's GRES data.
+ */
+class SlurmGresParser
+{
+
+    /**
+     * Parse requested generic resource (GRES) scheduling field from sacct.
+     *
+     * Expected ReqGRES format is a list of resources separated by commas with
+     * the each resource using the form name[:type:count]
+     *
+     * e.g. gpu:p100:2,bandwidth:1G
+     *
+     * @see https://slurm.schedmd.com/gres.html
+     *
+     * If the count (or anything else) contains pairs of parenthesis these will
+     * be removed along with everything contained between them.
+     *
+     * e.g. gpu:p100:2(IDX:0-1),hbm:0 is treated as gpu:p100:2,hbm:0
+     *
+     * @see https://github.com/PySlurm/pyslurm/issues/104
+     *
+     * @param string $gres ReqGRES field from sacct.
+     * @return array[] Parsed data. An array of arrays for each resource split
+     *     on ":".
+     */
+    public function parseReqGres($gres) {
+        if ($gres === '') {
+            return [];
+        }
+
+        // Remove anything contained in parenthesis along with the parenthesis.
+        $gres = preg_replace('/\(.*?\)/', '', $gres);
+
+        return array_map(
+            function ($resource) {
+                return explode(':', $resource);
+            },
+            explode(',', $gres)
+        );
+    }
+
+    /**
+     * Determine the GPU count from parsed ReqGRES data.
+     *
+     * @see \Xdmod\SlurmGresParser::parseReqGres
+     *
+     * @param array $gres Parsed ReqGRES data.
+     * @return int The GPU count.
+     */
+    public function getGpuCountFromGres(array $gres)
+    {
+        foreach ($gres as $resource) {
+            if ($resource[0] === 'gpu' && count($resource) > 1) {
+                return (int)$resource[count($resource) - 1];
+            }
+        }
+
+        return 0;
+    }
+}

configuration/datawarehouse.d/include/Jobs-sem-avg-gpu-agg.sql

@@ -0,0 +1,21 @@
+SQRT(
+    COALESCE(
+        (
+            (
+                SUM(POW(agg.gpu_count, 2) * agg.ended_job_count)
+                /
+                SUM(agg.ended_job_count)
+            )
+            -
+            POW(
+                SUM(agg.gpu_count * agg.ended_job_count)
+                /
+                SUM(agg.ended_job_count),
+                2
+            )
+        )
+        /
+        SUM(agg.ended_job_count),
+        0
+    )
+)

configuration/datawarehouse.d/include/Jobs-sem-avg-gpu-time.sql

@@ -0,0 +1,21 @@
+SQRT(
+    COALESCE(
+        (
+            (
+                SUM(POW(agg.gpu_count, 2) * agg.ended_job_count)
+                /
+                SUM(agg.running_job_count)
+            )
+            -
+            POW(
+                SUM(agg.gpu_count * agg.running_job_count)
+                /
+                SUM(agg.running_job_count),
+                2
+            )
+        )
+        /
+        SUM(agg.running_job_count),
+        0
+    )
+)

configuration/datawarehouse.d/ref/Jobs-group-bys.json

@@ -8,6 +8,9 @@
     "jobsize": {
         "$ref": "datawarehouse.d/ref/group-by-common.json#/jobsize"
     },
+    "gpucount": {
+        "$ref": "datawarehouse.d/ref/group-by-common.json#/gpucount"
+    },
     "jobwaittime": {
         "attribute_table": "job_wait_times",
         "attribute_table_schema": "modw",

configuration/datawarehouse.d/ref/Jobs-statistics.json

@@ -28,13 +28,28 @@
         "timeseries_formula": "COALESCE(SUM(agg.cpu_time) / SUM(agg.running_job_count), 0) / 3600.0",
         "unit": "CPU Hour"
     },
+    "avg_gpu_hours": {
+        "aggregate_formula": "COALESCE(SUM(agg.gpu_time) / SUM(CASE ${DATE_TABLE_ID_FIELD} WHEN ${MIN_DATE_ID} THEN agg.running_job_count ELSE agg.started_job_count END), 0) / 3600.0",
+        "description_html": "The average GPU hours (number of GPU cores x wall time hours) per ${ORGANIZATION_NAME} job.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours.<br/><i>Job: </i>A scheduled process for a computer resource in a batch processing environment.",
+        "name": "GPU Hours: Per Job",
+        "precision": 2,
+        "timeseries_formula": "COALESCE(SUM(agg.gpu_time) / SUM(agg.running_job_count), 0) / 3600.0",
+        "unit": "GPU Hour"
+    },
     "avg_job_size_weighted_by_cpu_hours": {
         "description_html": "The average ${ORGANIZATION_NAME} job size weighted by CPU Hours. Defined as <br><i>Average Job Size Weighted By CPU Hours: </i> sum(i = 0 to n){ job i core count * job i cpu hours}/sum(i =  0 to n){job i cpu hours}",
         "formula": "COALESCE(SUM(agg.processor_count * agg.cpu_time) / SUM(agg.cpu_time),0)",
         "name": "Job Size: Weighted By CPU Hours",
         "precision": 1,
         "unit": "Core Count"
     },
+    "avg_job_size_weighted_by_gpu_hours": {
+        "description_html": "The average ${ORGANIZATION_NAME} job size weighted by GPU Hours. Defined as <br><i>Average Job Size Weighted By GPU Hours: </i> sum(i = 0 to n){ job i core count * job i gpu hours}/sum(i =  0 to n){job i gpu hours}",
+        "formula": "COALESCE(SUM(agg.processor_count * agg.gpu_time) / SUM(agg.gpu_time), 0)",
+        "name": "Job Size: Weighted By GPU Hours",
+        "precision": 1,
+        "unit": "GPU Count"
+    },
     "avg_node_hours": {
         "aggregate_formula": "COALESCE(SUM(agg.node_time) / SUM(CASE ${DATE_TABLE_ID_FIELD} WHEN ${MIN_DATE_ID} THEN agg.running_job_count ELSE agg.started_job_count END), 0) / 3600.0",
         "description_html": "The average node hours (number of nodes x wall time hours) per ${ORGANIZATION_NAME} job.<br/><i>Job: </i>A scheduled process for a computer resource in a batch processing environment.",
@@ -51,6 +66,14 @@
         "timeseries_formula": "COALESCE(SUM(agg.processor_count * agg.running_job_count) / SUM(agg.running_job_count), 0)",
         "unit": "Core Count"
     },
+    "avg_gpus": {
+        "aggregate_formula": "COALESCE(SUM(agg.gpu_count * CASE ${DATE_TABLE_ID_FIELD} WHEN ${MIN_DATE_ID} THEN agg.running_job_count ELSE agg.started_job_count END) / SUM(CASE ${DATE_TABLE_ID_FIELD} WHEN ${MIN_DATE_ID} THEN agg.running_job_count ELSE agg.started_job_count END), 0)",
+        "description_html": "TThe average job size per  ${ORGANIZATION_NAME} job.<br><i>Job Size: </i>The number of GPUs used by a (parallel) job.",
+        "name": "GPU Count: Per Job",
+        "precision": 1,
+        "timeseries_formula": "COALESCE(SUM(agg.gpu_count * agg.running_job_count) / SUM(agg.running_job_count), 0)",
+        "unit": "GPU Count"
+    },
     "avg_waitduration_hours": {
         "description_html": "The average time, in hours, a ${ORGANIZATION_NAME} job waits before execution on the designated resource.<br/><i>Wait Time: </i>Wait time is defined as the linear time between submission of a job by a user until it begins to execute.",
         "formula": "COALESCE(SUM(agg.waitduration)/SUM(agg.started_job_count),0)/3600.0",
@@ -154,6 +177,19 @@
         },
         "unit": "Core Count"
     },
+    "sem_avg_gpus": {
+        "aggregate_formula": {
+            "$include": "datawarehouse.d/include/Jobs-sem-avg-gpu-agg.sql"
+        },
+        "description_html": "The standard error of the number of GPUs. <br/><i>Std Err of the Avg: </i> The standard deviation of the sample mean, estimated by the sample estimate of the population standard deviation (sample standard deviation) divided by the square root of the sample size (assuming statistical independence of the values in the sample).",
+        "name": "Std Dev: GPU Count: Per Job",
+        "precision": 2,
+        "show_in_catalog": false,
+        "timeseries_formula": {
+            "$include": "datawarehouse.d/include/Jobs-sem-avg-gpu-time.sql"
+        },
+        "unit": "GPU Count"
+    },
     "sem_avg_waitduration_hours": {
         "description_html": "The standard error of the average time, in hours, an ${ORGANIZATION_NAME} job had to wait until it began to execute.<br/><i>Std Err of the Avg: </i> The standard deviation of the sample mean, estimated by the sample estimate of the population standard deviation (sample standard deviation) divided by the square root of the sample size (assuming statistical independence of the values in the sample).",
         "formula": {
@@ -198,6 +234,13 @@
         "precision": 1,
         "unit": "CPU Hour"
     },
+    "total_gpu_hours": {
+        "description_html": "The total GPU hours (number of GPUs x wall time hours) used by ${ORGANIZATION_NAME} jobs.<br/>For each job, the GPU usage is aggregated. For example, if a job used 1000 GPUs for one minute, it would be aggregated as 1000 GPU minutes or 16.67 GPU hours.<br /><i>Job: </i>A scheduled process for a computer resource in a batch processing environment.",
+        "formula": "COALESCE(SUM(agg.gpu_time), 0) / 3600.0",
+        "name": "GPU Hours: Total",
+        "precision": 1,
+        "unit": "GPU Hour"
+    },
     "total_node_hours": {
         "description_html": "The total node hours (number of nodes x wall time hours) used by ${ORGANIZATION_NAME} jobs.<br/><i>Job: </i>A scheduled process for a computer resource in a batch processing environment.",
         "formula": "COALESCE(SUM(agg.node_time),0)/3600.0",

configuration/datawarehouse.d/ref/group-by-common.json

@@ -34,6 +34,41 @@
         "description_html": "A categorization of jobs into discrete groups based on the number of cores used by each job.",
         "name": "Job Size"
     },
+    "gpucount": {
+        "attribute_table": "gpu_buckets",
+        "attribute_table_schema": "modw",
+        "attribute_to_aggregate_table_key_map": [
+            {
+                "id": "gpubucket_id"
+            }
+        ],
+        "attribute_values_query": {
+            "joins": [
+                {
+                    "name": "gpu_buckets"
+                }
+            ],
+            "orderby": [
+                "id"
+            ],
+            "records": {
+                "id": "id",
+                "name": "description",
+                "order_id": "id",
+                "short_name": "description"
+            }
+        },
+        "category": "Administrative",
+        "chart_options": {
+            "dataset_display_type": {
+                "aggregate": "bar",
+                "timeseries": "area"
+            }
+        },
+        "data_sort_order": null,
+        "description_html": "A categorization of jobs into discrete groups based on the number of GPUs used by each job.",
+        "name": "GPU Count"
+    },
     "jobwalltime": {
         "attribute_table": "job_times",
         "attribute_table_schema": "modw",