-
Notifications
You must be signed in to change notification settings - Fork 68
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add GPU tracking to Jobs realm (#1270)
* Add GPU statistics and group by to Jobs realm * Refactor tests
- Loading branch information
Showing
858 changed files
with
16,639 additions
and
157 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
<?php | ||
/** | ||
* Slurm GRES parser. | ||
*/ | ||
|
||
namespace Xdmod; | ||
|
||
/** | ||
* Contains functions related to Slurm's GRES data. | ||
*/ | ||
class SlurmGresParser | ||
{ | ||
|
||
/** | ||
* Parse requested generic resource (GRES) scheduling field from sacct. | ||
* | ||
* Expected ReqGRES format is a list of resources separated by commas with | ||
* the each resource using the form name[:type:count] | ||
* | ||
* e.g. gpu:p100:2,bandwidth:1G | ||
* | ||
* @see https://slurm.schedmd.com/gres.html | ||
* | ||
* If the count (or anything else) contains pairs of parenthesis these will | ||
* be removed along with everything contained between them. | ||
* | ||
* e.g. gpu:p100:2(IDX:0-1),hbm:0 is treated as gpu:p100:2,hbm:0 | ||
* | ||
* @see https://github.com/PySlurm/pyslurm/issues/104 | ||
* | ||
* @param string $gres ReqGRES field from sacct. | ||
* @return array[] Parsed data. An array of arrays for each resource split | ||
* on ":". | ||
*/ | ||
public function parseReqGres($gres) { | ||
if ($gres === '') { | ||
return []; | ||
} | ||
|
||
// Remove anything contained in parenthesis along with the parenthesis. | ||
$gres = preg_replace('/\(.*?\)/', '', $gres); | ||
|
||
return array_map( | ||
function ($resource) { | ||
return explode(':', $resource); | ||
}, | ||
explode(',', $gres) | ||
); | ||
} | ||
|
||
/** | ||
* Determine the GPU count from parsed ReqGRES data. | ||
* | ||
* @see \Xdmod\SlurmGresParser::parseReqGres | ||
* | ||
* @param array $gres Parsed ReqGRES data. | ||
* @return int The GPU count. | ||
*/ | ||
public function getGpuCountFromGres(array $gres) | ||
{ | ||
foreach ($gres as $resource) { | ||
if ($resource[0] === 'gpu' && count($resource) > 1) { | ||
return (int)$resource[count($resource) - 1]; | ||
} | ||
} | ||
|
||
return 0; | ||
} | ||
} |
21 changes: 21 additions & 0 deletions
21
configuration/datawarehouse.d/include/Jobs-sem-avg-gpu-agg.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
SQRT( | ||
COALESCE( | ||
( | ||
( | ||
SUM(POW(agg.gpu_count, 2) * agg.ended_job_count) | ||
/ | ||
SUM(agg.ended_job_count) | ||
) | ||
- | ||
POW( | ||
SUM(agg.gpu_count * agg.ended_job_count) | ||
/ | ||
SUM(agg.ended_job_count), | ||
2 | ||
) | ||
) | ||
/ | ||
SUM(agg.ended_job_count), | ||
0 | ||
) | ||
) |
21 changes: 21 additions & 0 deletions
21
configuration/datawarehouse.d/include/Jobs-sem-avg-gpu-time.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
SQRT( | ||
COALESCE( | ||
( | ||
( | ||
SUM(POW(agg.gpu_count, 2) * agg.ended_job_count) | ||
/ | ||
SUM(agg.running_job_count) | ||
) | ||
- | ||
POW( | ||
SUM(agg.gpu_count * agg.running_job_count) | ||
/ | ||
SUM(agg.running_job_count), | ||
2 | ||
) | ||
) | ||
/ | ||
SUM(agg.running_job_count), | ||
0 | ||
) | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.