added py-spy support

INSTALL

@@ -151,6 +151,10 @@ make install
    requires Dyninst as a dependence even when using the GDB
    backend.
 
+   STAT has the ability to use py-spy to gather python stack
+   traces. To specify the path to py-spy, use the --with-pyspy
+   flag.
+
    Dyninst 12 removed the local_var.h header file that STAT uses
    to gather python traces. If this feature is needed, you will
    either need to use Dyninst version 11 or lower, or copy the

config/x_ac_gdb.m4

@@ -9,7 +9,7 @@ AC_DEFUN([X_AC_GDB], [
 
   AC_ARG_WITH(gdb,
     [AS_HELP_STRING([--with-gdb=path],
-      [Use cmd as the default remote shell]
+      [Use path for gdb backend]
     )],
     [withgdb="$withval"
       AM_CONDITIONAL([ENABLE_GDB], true)

config/x_ac_python.m4

@@ -50,4 +50,15 @@ AC_DEFUN([X_AC_PYTHON], [
   fi
   AC_MSG_RESULT($python_version)
   AM_COND_IF([ENABLE_GDB], [BELIBS="-lpython$python_version $BELIBS"], [])
+
+  AC_ARG_WITH(pyspy,
+    [AS_HELP_STRING([--with-pyspy=path],
+      [Use path for py-spy backend]
+    )],
+    [withpyspy="$withval"
+      AM_CONDITIONAL([ENABLE_PYSPY], true)
+    ],
+    [withpyspy=pyspy]
+  )
+  AC_PATH_PROG(pyspycmd,$withpyspy,[py-spy],/,/usr/local/bin:/usr/bin:/bin/usr/local/bin)
 ])

configure.ac

@@ -3,7 +3,7 @@
 
 AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2007-2020, Lawrence Livermore National Security, LLC.])
-AC_INIT([STAT], [4.2.1])
+AC_INIT([STAT], [4.2.2])
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_SRCDIR([configure.ac])
 AC_CONFIG_HEADER([config.h])

doc/src/stat_changelog.sgml

@@ -1,3 +1,17 @@
+<section>
+  <title>stat version 4.2.2</title>
+  <para>
+    <itemizedlist>
+      <listitem><para>
+        Added gdb-oneapi backend support
+      </para></listitem>
+      <listitem><para>
+        Added py-spy backend support
+      </para></listitem>
+    </itemizedlist>
+  </para>
+</section>
+
 <section>
   <title>stat version 4.2.1</title>
   <para>

doc/src/stat_installation.sgml

@@ -52,6 +52,10 @@ STAT will also, by default, add rpaths to dependent libraries.  This behavior ca
 STAT can also be configured to use GDB as a backend instead of Dyninst. To specify the path to gdb, use the <option>--with-gdb</option> flag. If CUDA kernal traces are desired, the path should point to a working cuda-gdb executable. Note that STAT currently still requires Dyninst as a dependence even when using the GDB backend.
 </para>
 
+<para>
+STAT has the ability to use py-spy to gather python stack traces. To specify the path to py-spy, use the <option>--with-pyspy</option> flag.
+</para>
+
 <para>
 Dyninst 12 removed the local_var.h header file that STAT uses to gather python traces. If this feature is needed, you will either need to use Dyninst version 11 or lower, or copy the local_var.h file from <ulink url="https://raw.githubusercontent.com/dyninst/dyninst/5c7e0ee327399cfae50d77f977ff6655c2ca3ae4/stackwalk/h/local_var.h"></ulink> to your Dyninst 12+ installation's include directory
 </para>

doc/src/stat_options.sgml

@@ -295,4 +295,12 @@
         </para>
     </listitem>
 </varlistentry>
+<varlistentry>
+    <term>-Y, --pyspy</term>
+    <listitem>
+        <para>
+        Use py-spy backend to gather python stack traces.
+        </para>
+    </listitem>
+</varlistentry>
 </variablelist>

doc/src/stat_preference_files.sgml

@@ -291,4 +291,20 @@ Here is a list of options:
         </para>
     </listitem>
 </varlistentry>
+<varlistentry>
+    <term>Use PySpy = <replaceable class="parameter">true|false</replaceable></term>
+    <listitem>
+        <para>
+        Controls whether to use py-spy to drive the deamons.
+        </para>
+    </listitem>
+</varlistentry>
+<varlistentry>
+    <term>PySpy Path = <replaceable class="parameter">path</replaceable></term>
+    <listitem>
+        <para>
+        Use the py-spy executable installed in <replaceable class="parameter">path</replaceable> for debugging when Use PySpy is set to true.
+        </para>
+    </listitem>
+</varlistentry>
 </variablelist>

doc/src/statgui_options.sgml

@@ -135,4 +135,12 @@
         </para>
     </listitem>
 </varlistentry>
+<varlistentry>
+    <term>-Y, --pyspy</term>
+    <listitem>
+        <para>
+        Use py-spy backend to gather python traces.
+        </para>
+    </listitem>
+</varlistentry>
 </variablelist>

examples/scripts/python_test.py

@@ -1,11 +1,42 @@
-print 'job starting'
-import time
-for i in range(12):
-    time.sleep(5)
-print 'done sleeping, importing parser'
-import parser
-print 'importing socket'
-import socket
-print 'sleeping'
-time.sleep(15)
-print 'done'
+has_mpi4py = True
+try:
+    from mpi4py import MPI
+except:
+    has_mpi4py = false
+
+def foo(rank):
+    if rank == 0:
+        bar(rank)
+    else:
+        foo(rank - 1)
+
+def foo2(rank):
+    if rank == 0:
+        bar(rank)
+    else:
+        foo(rank - 1)
+
+def bar(rank):
+    import time
+    for i in range(120):
+        time.sleep(5)
+    print('done sleeping, importing parser')
+    import parser
+    print ('importing socket')
+    import socket
+    print ('sleeping')
+    time.sleep(15)
+
+rank = 0
+if has_mpi4py == True:
+    comm = MPI.COMM_WORLD
+    rank = comm.Get_rank()
+    size = comm.Get_size()
+if rank == 0:
+    print('job starting')
+if rank % 2 == 0:
+    foo(rank)
+else:
+    foo2(rank)
+if rank == 0:
+    print ('done')

man/STAT.1

@@ -144,6 +144,9 @@ Use (cuda-)gdb to drive the daemons. If you are using cuda-gdb and want stack tr
 .TP
 \fB-Q, --cudaquick\fR
 When using cuda-gdb as the BE, gather less comprehensive, but faster cuda traces. Cuda frames will only show the top of the stack, not the full call path. This also defaults to display filename and line number and will not resolve the function name.
+.TP
+\fB-Y, --pyspy\fR
+Use py-spy backend to gather python stack traces.
 .SH "EXAMPLE"
 .PP
 The most typical usage is to invoke STAT on the job launcher's PID:

man/STATGUI.1

@@ -98,6 +98,9 @@ Use (cuda-)gdb to drive the daemons. If you are using cuda-gdb and want stack tr
 .TP
 \fB-Q, --cudaquick\fR
 When using cuda-gdb as the BE, gather less comprehensive, but faster cuda traces. Cuda frames will only show the top of the stack, not the full call path. This also defaults to display filename and line number and will not resolve the function name.
+.TP
+\fB-Y, --pyspy\fR
+Use py-spy backend to gather python traces.
 .SH "AUTHOR"
 .PP
 (Written by ) Gregory  L.  Lee  

scripts/DysectView.py

@@ -22,7 +22,7 @@
 __author__ = ["Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"] "Jesper Puge Nielsen"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import STATview

scripts/Makefile.am

@@ -4,6 +4,7 @@ python_PYTHON = STATmain.py stat_merge_base.py STATmerge.py core_file_merger.py
 if ENABLE_GDB
 python_PYTHON += stat_cuda_gdb.py cuda_gdb.py gdb.py roc_gdb.py oneapi_gdb.py
 endif
+python_PYTHON += stat_py_spy.py
 if ENABLE_GUI
 python_PYTHON += STATGUI.py STATview.py
 if ENABLE_PYTHON2

scripts/STATGUI.in

@@ -22,6 +22,9 @@ fi
 if test -z "$STAT_GDB" ; then
     export STAT_GDB=@gdbcmd@
 fi
+if test -z "$STAT_PYSPY" ; then
+    export STAT_PYSPY=@pyspycmd@
+fi
 if test -z "$STAT_FGFS_FILTER_PATH" ; then
     if test -n "$FGFS_PREFIX" ; then
         export STAT_FGFS_FILTER_PATH=$FGFS_PREFIX/lib/libfgfs_filter.so

scripts/STATGUI.py

@@ -47,7 +47,7 @@
     new_dlflags = ctypes.RTLD_GLOBAL | dlflags
     sys.setdlopenflags(new_dlflags)
 
-from STAT import STAT_FrontEnd, intArray, STAT_LOG_NONE, STAT_LOG_FE, STAT_LOG_BE, STAT_LOG_CP, STAT_LOG_MRN, STAT_LOG_SW, STAT_LOG_SWERR, STAT_OK, STAT_APPLICATION_EXITED, STAT_VERBOSE_ERROR, STAT_VERBOSE_FULL, STAT_VERBOSE_STDOUT, STAT_TOPOLOGY_AUTO, STAT_TOPOLOGY_DEPTH, STAT_TOPOLOGY_FANOUT, STAT_TOPOLOGY_USER, STAT_PENDING_ACK, STAT_LAUNCH, STAT_ATTACH, STAT_SERIAL_ATTACH, STAT_GDB_ATTACH, STAT_SERIAL_GDB_ATTACH, STAT_SAMPLE_FUNCTION_ONLY, STAT_SAMPLE_LINE, STAT_SAMPLE_PC, STAT_SAMPLE_COUNT_REP, STAT_SAMPLE_THREADS, STAT_SAMPLE_CLEAR_ON_SAMPLE, STAT_SAMPLE_PYTHON, STAT_SAMPLE_MODULE_OFFSET, STAT_CP_NONE, STAT_CP_SHAREAPPNODES, STAT_CP_EXCLUSIVE
+from STAT import STAT_FrontEnd, intArray, STAT_LOG_NONE, STAT_LOG_FE, STAT_LOG_BE, STAT_LOG_CP, STAT_LOG_MRN, STAT_LOG_SW, STAT_LOG_SWERR, STAT_OK, STAT_APPLICATION_EXITED, STAT_VERBOSE_ERROR, STAT_VERBOSE_FULL, STAT_VERBOSE_STDOUT, STAT_TOPOLOGY_AUTO, STAT_TOPOLOGY_DEPTH, STAT_TOPOLOGY_FANOUT, STAT_TOPOLOGY_USER, STAT_PENDING_ACK, STAT_LAUNCH, STAT_ATTACH, STAT_SERIAL_ATTACH, STAT_GDB_ATTACH, STAT_SERIAL_GDB_ATTACH, STAT_PYSPY_ATTACH, STAT_SERIAL_PYSPY_ATTACH, STAT_SAMPLE_FUNCTION_ONLY, STAT_SAMPLE_LINE, STAT_SAMPLE_PC, STAT_SAMPLE_COUNT_REP, STAT_SAMPLE_THREADS, STAT_SAMPLE_CLEAR_ON_SAMPLE, STAT_SAMPLE_PYTHON, STAT_SAMPLE_MODULE_OFFSET, STAT_CP_NONE, STAT_CP_SHAREAPPNODES, STAT_CP_EXCLUSIVE
 HAVE_OPENMP_SUPPORT = True
 try:
     from STAT import STAT_SAMPLE_OPENMP
@@ -202,6 +202,10 @@ def __init__(self, args):
             gdb_path = os.environ['STAT_GDB']
         except:
             gdb_path = 'gdb'
+        try:
+            pyspy_path = os.environ['STAT_PYSPY']
+        except:
+            pyspy_path = 'py-spy'
         options = {'Remote Host':                      "localhost",
                    'Remote Host Shell':                "ssh",
                    'Serial Remote Host Shell':         "ssh",
@@ -233,6 +237,9 @@ def __init__(self, args):
                    'Use MRNet Printf':                 False,
                    'GDB BE':                           False,
                    'GDB Path':                         gdb_path,
+                   'PySpy BE':                         False,
+                   'PySpy Path':                       pyspy_path,
+                   'Use PySpy':                        False,
                    'With CUDA Quick':                  False,
                    'Verbosity Type':                   'error',
                    'Communication Nodes':              '',
@@ -343,6 +350,8 @@ def __init__(self, args):
                     self.options['GDB BE'] = True
                 if args.cudaquick is True:
                     self.options['With CUDA Quick'] = True
+            if args.pyspy is True:
+                self.options['Use PySpy'] = True
             if args.log is not None:
                 if 'CP' in args.log:
                     self.options['Log CP'] = True
@@ -413,6 +422,8 @@ def __init__(self, args):
                 if HAVE_GDB_SUPPORT:
                     if args.gdb is not None:
                         self.options['GDB BE'] = True
+                if args.pyspy is not None:
+                    self.options['Use PySpy'] = True
                 stat_wait_dialog.show_wait_dialog_and_run(self.attach_cb, (None, False, False, STAT_ATTACH), self.attach_task_list)
                 return
             elif args.serial is not None:
@@ -558,7 +569,7 @@ def on_properties(self, action):
         frame.add(text_view)
         vbox.pack_start(frame, False, False, 0)
 
-        if self.STAT.getApplicationOption() != STAT_SERIAL_ATTACH and self.STAT.getApplicationOption() != STAT_SERIAL_GDB_ATTACH:
+        if self.STAT.getApplicationOption() != STAT_SERIAL_ATTACH and self.STAT.getApplicationOption() != STAT_SERIAL_GDB_ATTACH and self.STAT.getApplicationOption() != STAT_SERIAL_PYSPY_ATTACH:
             frame = gtk.Frame(label='Job Launcher (host:PID)')
             text_view = gtk.TextView()
             text_buffer = gtk.TextBuffer()
@@ -988,6 +999,8 @@ def on_reattach(self, action):
                 self.STAT.addSerialProcess(process)
             if self.options['GDB BE'] is True:
                 self.attach_cb(None, False, True, STAT_SERIAL_GDB_ATTACH)
+            elif self.options['Use PySpy'] is True:
+                self.attach_cb(None, False, True, STAT_SERIAL_PYSPY_ATTACH)
             else:
                 self.attach_cb(None, False, True, STAT_SERIAL_ATTACH)
         else:
@@ -1227,6 +1240,8 @@ def on_attach(self, action):
         if HAVE_GDB_SUPPORT:
             self.pack_check_button(vbox2, 'GDB BE')
             self.pack_string_option(vbox2, 'GDB Path', attach_dialog)
+        self.pack_check_button(vbox2, 'Use PySpy')
+        self.pack_string_option(vbox2, 'PySpy Path', attach_dialog)
         frame.add(vbox2)
         vbox.pack_start(frame, False, False, 5)
         frame = gtk.Frame(label='Debug Logs')
@@ -1280,6 +1295,8 @@ def attach_serial_processes_cb(self, entry=None, attach_dialog=None, in_processe
             self.STAT.addSerialProcess(process)
         if self.options['GDB BE'] is True:
             self.attach_cb(attach_dialog, False, True, STAT_SERIAL_GDB_ATTACH)
+        elif self.options['Use PySpy'] is True:
+            self.attach_cb(attach_dialog, False, True, STAT_SERIAL_PYSPY_ATTACH)
         else:
             self.attach_cb(attach_dialog, False, True, STAT_SERIAL_ATTACH)
         return True
@@ -1386,7 +1403,10 @@ def attach_cb(self, attach_dialog, launch, serial, application_option=STAT_ATTAC
         self.STAT.setToolDaemonExe(self.options['Tool Daemon Path'])
         self.STAT.setFilterPath(self.options['Filter Path'])
         os.environ['STAT_GDB'] = self.options['GDB Path']
+        os.environ['STAT_PYSPY'] = self.options['PySpy Path']
         log_type = STAT_LOG_NONE
+        if self.options['Use PySpy']and application_option != STAT_SERIAL_PYSPY_ATTACH:
+            application_option = STAT_PYSPY_ATTACH
         if self.options['GDB BE'] and application_option != STAT_SERIAL_GDB_ATTACH:
             application_option = STAT_GDB_ATTACH
         if self.options['Log Frontend']:
@@ -1474,7 +1494,7 @@ def attach_cb(self, attach_dialog, launch, serial, application_option=STAT_ATTAC
             self.on_fatal_error()
             return False
 
-        if application_option != STAT_SERIAL_ATTACH and application_option != STAT_SERIAL_GDB_ATTACH:
+        if application_option != STAT_SERIAL_ATTACH and application_option != STAT_SERIAL_GDB_ATTACH and application_option != STAT_SERIAL_PYSPY_ATTACH:
             while 1:
                 run_gtk_main_loop()
                 ret = self.STAT.connectMrnetTree(False)
@@ -2338,7 +2358,7 @@ def cancel_entry(dialog):
             arg_list.append(filepath)
             cli_attach = False
             if self.STAT is not None:
-                if self.STAT.getApplicationOption() == STAT_SERIAL_ATTACH or self.STAT.getApplicationOption() != STAT_SERIAL_GDB_ATTACH:
+                if self.STAT.getApplicationOption() == STAT_SERIAL_ATTACH or self.STAT.getApplicationOption() != STAT_SERIAL_GDB_ATTACH or self.STAT.getApplicationOption() != STAT_SERIAL_PYSPY_ATTACH:
                     cli_attach = True
             if cli_attach is True:
                 arg_list.append('-e')

scripts/STATmain.py

@@ -97,6 +97,7 @@ def STATmain_main(in_arg_list, command=None):
         gui_parser.add_argument("-o", "--withopenmp", help="translate OpenMP stacks to logical application view", action="store_true")
         gui_parser.add_argument("-w", "--withthreads", help="sample helper threads in addition to the main thread", action="store_true")
         gui_parser.add_argument("-y", "--pythontrace", help="gather Python script level stack traces", action="store_true")
+        gui_parser.add_argument("-Y", "--pyspy", help="gather Python script level stack traces using py-spy", action="store_true")
         gui_parser.add_argument("-U", "--countrep", help="only gather count and a single representative", action="store_true")
         gui_parser.add_argument("-d", "--debugdaemons", help="launch the daemons under the debugger", action="store_true")
         gui_parser.add_argument("-L", "--logdir", help="logging output directory")

scripts/attach_helper.py

@@ -22,7 +22,7 @@
 __author__ = ["Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import subprocess

scripts/bg_core_backtrace.py

@@ -21,7 +21,7 @@
 __author__ = ["Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import sys

scripts/core_file_merger.py

@@ -30,7 +30,7 @@
 __author__ = ["Dane Gardner", "Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 ###############################################################################

scripts/cuda_gdb.py

@@ -22,7 +22,7 @@
 __author__ = ["Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import sys

scripts/gdb.py

@@ -22,7 +22,7 @@
 __author__ = ["Gregory Lee <lee218@llnl.gov>", "Dorian Arnold", "Matthew LeGendre", "Dong Ahn", "Bronis de Supinski", "Barton Miller", "Martin Schulz", "Niklas Nielson", "Nicklas Bo Jensen", "Jesper Nielson"]
 __version_major__ = 4
 __version_minor__ = 2
-__version_revision__ = 1
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import subprocess

scripts/oneapi_gdb.py

@@ -23,7 +23,7 @@
 __author__ = ["M. Oguzhan Karakaya <oguzhan.karakaya@intel.com>"]
 __version_major__ = 4
 __version_minor__ = 1
-__version_revision__ = 0
+__version_revision__ = 2
 __version__ = "%d.%d.%d" %(__version_major__, __version_minor__, __version_revision__)
 
 import sys