From 931646a51c4f890f6b797cd8f5d8a0dc425366e1 Mon Sep 17 00:00:00 2001 From: Johannes Elmsheuser Date: Mon, 29 Apr 2024 11:22:36 +0200 Subject: [PATCH 1/6] Update output parsing for nvidiamon --- package/src/nvidiamon.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package/src/nvidiamon.cpp b/package/src/nvidiamon.cpp index 04c538a..828a681 100644 --- a/package/src/nvidiamon.cpp +++ b/package/src/nvidiamon.cpp @@ -74,22 +74,22 @@ void nvidiamon::update_stats(const std::vector& pids, } // Loop over output - unsigned int gpu_idx{}, sm{}, mem{}, fb_mem{}; + unsigned int gpu_idx{}, sm{}, mem{}, fb_mem{}, ccpm{}; pid_t pid{}; - std::string enc{}, dec{}, cg_type{}, cmd_name{}; + std::string enc{}, dec{}, jpg{}, ofa{}, cg_type{}, cmd_name{}; std::unordered_map activegpus{}; // Avoid double counting active GPUs for (const auto& s : cmd_result.second) { if (s[0] == '#') continue; std::istringstream instr(s); - instr >> gpu_idx >> pid >> cg_type >> sm >> mem >> enc >> dec >> fb_mem >> - cmd_name; + instr >> gpu_idx >> pid >> cg_type >> sm >> mem >> enc >> dec >> jpg >> ofa >> fb_mem >> + ccpm >> cmd_name; auto read_ok = !(instr.fail() || instr.bad()); // eof() is ok if (read_ok) { if (log_level <= spdlog::level::debug) { std::stringstream strm; strm << "Good read: " << gpu_idx << " " << pid << " " << cg_type << " " - << sm << " " << mem << " " << enc << " " << dec << " " << fb_mem + << sm << " " << mem << " " << enc << " " << dec << " " << jpg << " " << ofa << " " << fb_mem << " " << ccpm << " " << cmd_name << std::endl; debug(strm.str()); } @@ -115,7 +115,7 @@ void nvidiamon::update_stats(const std::vector& pids, std::stringstream strm; strm << "Bad read of line: " << s << std::endl; strm << "Parsed to: " << gpu_idx << " " << pid << " " << cg_type << " " - << sm << " " << mem << " " << enc << " " << dec << " " << fb_mem + << sm << " " << mem << " " << enc << " " << dec << " " << jpg << " " << ofa << " " << fb_mem << " " << ccpm << " " << cmd_name << std::endl; strm << "StringStream status: good()=" << instr.good(); From 52671cf0e5ea0ed0fa92d77250d56cafde255b90 Mon Sep 17 00:00:00 2001 From: Graeme Stewart Date: Fri, 3 May 2024 12:19:42 +0200 Subject: [PATCH 2/6] Update test files for nvidia-smi parsing Update to the new nvidia-smi pmon output fields Add a pycuda GPU burner script for tests --- README.md | 2 +- package/scripts/gpu-burner.py | 38 +++++++++++++++++++ package/scripts/precook_test.py | 13 ++++--- .../scripts/precooked_tests/drop/1/nvidia/smi | 2 +- .../scripts/precooked_tests/drop/2/nvidia/smi | 2 +- .../scripts/precooked_tests/drop/3/nvidia/smi | 2 +- 6 files changed, 50 insertions(+), 9 deletions(-) create mode 100644 package/scripts/gpu-burner.py diff --git a/README.md b/README.md index 00746b4..1cf45bc 100644 --- a/README.md +++ b/README.md @@ -254,4 +254,4 @@ to CMake using `Gperftools_ROOT_DIR`. # Copyright -Copyright (c) 2018-2023 CERN. +Copyright (c) 2018-2024 CERN. diff --git a/package/scripts/gpu-burner.py b/package/scripts/gpu-burner.py new file mode 100644 index 0000000..820f1f6 --- /dev/null +++ b/package/scripts/gpu-burner.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python +# +# This is a slightly adapted "hello, world" script from +# pycuda, that can be used for stressing a CUDA GPU for +# tests +# +# pycuda is required! +# + +import pycuda.autoinit +import pycuda.driver as drv +import numpy +from time import time + +from pycuda.compiler import SourceModule +mod = SourceModule(""" +__global__ void multiply_them(float *dest, float *a, float *b, float *c) +{ + const int i = threadIdx.x; + dest[i] = a[i] * b[i] + c[i]; +} +""") + +multiply_them = mod.get_function("multiply_them") + +a = numpy.random.randn(1024).astype(numpy.float32) +b = numpy.random.randn(1024).astype(numpy.float32) +c = numpy.random.randn(1024).astype(numpy.float32) + +dest = numpy.zeros_like(a) + +start = time() +while (time() - start < 20): + multiply_them( + drv.Out(dest), drv.In(a), drv.In(b), drv.In(c), + block=(1024,1,1), grid=(1,1)) + +print(dest-a*b+c) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 7a9f52e..16b857e 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -96,7 +96,7 @@ def make_net(proc_net, fixed_value, rand=False): def make_nvidia(proc_nvidia, fixed_value, rand=False): # idx smi_fname = os.path.join(proc_nvidia, "smi") - memory_lim = 10000 + memory_lim = 100 with open(smi_fname, "w") as f: params = [ 0, # idx @@ -104,10 +104,13 @@ def make_nvidia(proc_nvidia, fixed_value, rand=False): "G", # type random.randint(0, memory_lim) if rand else fixed_value, # sm random.randint(0, memory_lim) if rand else fixed_value, # mem - # enc, dec are not monitored metrics - 0, # enc - 0, # dec - random.randint(0, memory_lim) if rand else fixed_value, # fb + # The following are not monitored metrics + "-", # enc + "-", # dec + "-", # jpg + "-", # ofa + random.randint(0, memory_lim*100) if rand else fixed_value, # fb + 0, # ccpm "python3", # command ] for param in params: diff --git a/package/scripts/precooked_tests/drop/1/nvidia/smi b/package/scripts/precooked_tests/drop/1/nvidia/smi index 2e17e65..545520b 100644 --- a/package/scripts/precooked_tests/drop/1/nvidia/smi +++ b/package/scripts/precooked_tests/drop/1/nvidia/smi @@ -1 +1 @@ -0 1729 G 50 50 0 0 50 python3 +0 1729 G 86 53 - - - - 9077 0 python3 diff --git a/package/scripts/precooked_tests/drop/2/nvidia/smi b/package/scripts/precooked_tests/drop/2/nvidia/smi index e27a02f..b6efbac 100644 --- a/package/scripts/precooked_tests/drop/2/nvidia/smi +++ b/package/scripts/precooked_tests/drop/2/nvidia/smi @@ -1 +1 @@ -0 1729 G 100 100 0 0 100 python3 +0 1729 G 17 93 - - - - 4374 0 python3 diff --git a/package/scripts/precooked_tests/drop/3/nvidia/smi b/package/scripts/precooked_tests/drop/3/nvidia/smi index 9dc7ae5..123136a 100644 --- a/package/scripts/precooked_tests/drop/3/nvidia/smi +++ b/package/scripts/precooked_tests/drop/3/nvidia/smi @@ -1 +1 @@ -0 1729 G 20 20 0 0 20 python3 +0 1729 G 2 69 - - - - 4629 0 python3 From ccdd02bfe137be9ba0401dc7e1efcd02cb0da332 Mon Sep 17 00:00:00 2001 From: Graeme Stewart Date: Fri, 3 May 2024 12:46:09 +0200 Subject: [PATCH 3/6] Parse ccpm field as string This can be a "-" instead of 0 --- package/src/nvidiamon.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package/src/nvidiamon.cpp b/package/src/nvidiamon.cpp index 828a681..947c65c 100644 --- a/package/src/nvidiamon.cpp +++ b/package/src/nvidiamon.cpp @@ -74,9 +74,9 @@ void nvidiamon::update_stats(const std::vector& pids, } // Loop over output - unsigned int gpu_idx{}, sm{}, mem{}, fb_mem{}, ccpm{}; + unsigned int gpu_idx{}, sm{}, mem{}, fb_mem{}; pid_t pid{}; - std::string enc{}, dec{}, jpg{}, ofa{}, cg_type{}, cmd_name{}; + std::string enc{}, dec{}, jpg{}, ofa{}, cg_type{}, ccpm{}, cmd_name{}; std::unordered_map activegpus{}; // Avoid double counting active GPUs for (const auto& s : cmd_result.second) { From 53ce2053a3f6429ed94c2385d18e2d6086ebef62 Mon Sep 17 00:00:00 2001 From: Graeme Stewart Date: Fri, 3 May 2024 12:47:15 +0200 Subject: [PATCH 4/6] Update precook script and precooked ouptuts Ensure precooked values are fixed to what we want --- package/scripts/precook_test.py | 10 ++++++---- package/scripts/precooked_tests/drop/1/nvidia/smi | 2 +- package/scripts/precooked_tests/drop/2/nvidia/smi | 2 +- package/scripts/precooked_tests/drop/3/nvidia/smi | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 16b857e..3a1fc34 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -95,21 +95,23 @@ def make_net(proc_net, fixed_value, rand=False): def make_nvidia(proc_nvidia, fixed_value, rand=False): # idx + print(proc_nvidia, fixed_value, rand) smi_fname = os.path.join(proc_nvidia, "smi") - memory_lim = 100 + pct_lim = 100 + memory_lim = 10000 with open(smi_fname, "w") as f: params = [ 0, # idx pid, # pid "G", # type - random.randint(0, memory_lim) if rand else fixed_value, # sm - random.randint(0, memory_lim) if rand else fixed_value, # mem + random.randint(0, pct_lim) if rand else fixed_value, # sm + random.randint(0, pct_lim) if rand else fixed_value, # mem # The following are not monitored metrics "-", # enc "-", # dec "-", # jpg "-", # ofa - random.randint(0, memory_lim*100) if rand else fixed_value, # fb + random.randint(0, memory_lim) if rand else fixed_value, # fb 0, # ccpm "python3", # command ] diff --git a/package/scripts/precooked_tests/drop/1/nvidia/smi b/package/scripts/precooked_tests/drop/1/nvidia/smi index 545520b..835afdf 100644 --- a/package/scripts/precooked_tests/drop/1/nvidia/smi +++ b/package/scripts/precooked_tests/drop/1/nvidia/smi @@ -1 +1 @@ -0 1729 G 86 53 - - - - 9077 0 python3 +0 1729 G 50 50 - - - - 50 0 python3 diff --git a/package/scripts/precooked_tests/drop/2/nvidia/smi b/package/scripts/precooked_tests/drop/2/nvidia/smi index b6efbac..b9e438f 100644 --- a/package/scripts/precooked_tests/drop/2/nvidia/smi +++ b/package/scripts/precooked_tests/drop/2/nvidia/smi @@ -1 +1 @@ -0 1729 G 17 93 - - - - 4374 0 python3 +0 1729 G 100 100 - - - - 100 0 python3 diff --git a/package/scripts/precooked_tests/drop/3/nvidia/smi b/package/scripts/precooked_tests/drop/3/nvidia/smi index 123136a..30220eb 100644 --- a/package/scripts/precooked_tests/drop/3/nvidia/smi +++ b/package/scripts/precooked_tests/drop/3/nvidia/smi @@ -1 +1 @@ -0 1729 G 2 69 - - - - 4629 0 python3 +0 1729 G 20 20 - - - - 20 0 python3 From b26bddd3f378d3b0bb1793e04f168d969bd5a321 Mon Sep 17 00:00:00 2001 From: Graeme Stewart Date: Fri, 3 May 2024 12:56:27 +0200 Subject: [PATCH 5/6] Fix hash-bang an mode on GPU burner --- package/scripts/gpu-burner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 package/scripts/gpu-burner.py diff --git a/package/scripts/gpu-burner.py b/package/scripts/gpu-burner.py old mode 100644 new mode 100755 index 820f1f6..493e204 --- a/package/scripts/gpu-burner.py +++ b/package/scripts/gpu-burner.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # # This is a slightly adapted "hello, world" script from # pycuda, that can be used for stressing a CUDA GPU for From 7fff4d8fbfa9bf7f757b94f0e8dd7ce151e2d53c Mon Sep 17 00:00:00 2001 From: Graeme Stewart Date: Fri, 3 May 2024 14:38:18 +0200 Subject: [PATCH 6/6] Python reformatting With latest verisons of black and flake8 There is one import in gpu-burner.py that is needed (pycuda.autoinit) as it has side effects, so this is marked as excempt for flake8 --- package/scripts/gpu-burner.py | 17 ++++++++++------- package/scripts/precook_test.py | 2 +- package/scripts/prmon_compress_output.py | 8 +++++--- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/package/scripts/gpu-burner.py b/package/scripts/gpu-burner.py index 493e204..bc51372 100755 --- a/package/scripts/gpu-burner.py +++ b/package/scripts/gpu-burner.py @@ -7,19 +7,22 @@ # pycuda is required! # -import pycuda.autoinit +import pycuda.autoinit # noqa: F401 import pycuda.driver as drv import numpy from time import time from pycuda.compiler import SourceModule -mod = SourceModule(""" + +mod = SourceModule( + """ __global__ void multiply_them(float *dest, float *a, float *b, float *c) { const int i = threadIdx.x; dest[i] = a[i] * b[i] + c[i]; } -""") +""" +) multiply_them = mod.get_function("multiply_them") @@ -30,9 +33,9 @@ dest = numpy.zeros_like(a) start = time() -while (time() - start < 20): +while time() - start < 20: multiply_them( - drv.Out(dest), drv.In(a), drv.In(b), drv.In(c), - block=(1024,1,1), grid=(1,1)) + drv.Out(dest), drv.In(a), drv.In(b), drv.In(c), block=(1024, 1, 1), grid=(1, 1) + ) -print(dest-a*b+c) +print(dest - a * b + c) diff --git a/package/scripts/precook_test.py b/package/scripts/precook_test.py index 3a1fc34..131006b 100755 --- a/package/scripts/precook_test.py +++ b/package/scripts/precook_test.py @@ -112,7 +112,7 @@ def make_nvidia(proc_nvidia, fixed_value, rand=False): "-", # jpg "-", # ofa random.randint(0, memory_lim) if rand else fixed_value, # fb - 0, # ccpm + 0, # ccpm "python3", # command ] for param in params: diff --git a/package/scripts/prmon_compress_output.py b/package/scripts/prmon_compress_output.py index ef4daf5..6ef7801 100755 --- a/package/scripts/prmon_compress_output.py +++ b/package/scripts/prmon_compress_output.py @@ -122,9 +122,11 @@ def main(): parser.add_argument( "--precision", - type=lambda x: float(x) - if 0 < float(x) < 1 - else parser.exit(-1, "Precision must be strictly between 0 and 1"), + type=lambda x: ( + float(x) + if 0 < float(x) < 1 + else parser.exit(-1, "Precision must be strictly between 0 and 1") + ), default=0.05, help="precision value for interpolation threshold", )