Merge pull request #200 from jpwhite4/release_prep

Update version info for 1.2 release

CHANGELOG.md

@@ -1,5 +1,31 @@
 # Changelog
 
+## [1.2.0] - 2019-04-24
+
+### Added
+- Added GPU power plugin that summarizes the power usage for nvidia GPUs.
+- Added IPMI power plugin that summarizes the power usage from impi.
+- Added support for scanning the YYYY/MM/HOSTNAME/YYYY-MM-DD directory format for PCP archives.
+- Added `--dry-run` option to `indexarchives.py` script (used for testing purposes).
+
+### Changed
+- Removed deprecated code that supported python 2.6.
+- Updated templates to use the new recommended directory format.
+- Updated test harness to work with current plugins.
+- Database access code now sets the `local_infile` flag (needed for MySQL driver version 2.0.0 or later).
+
+### Fixed
+- Summarization script now only includes job start and end archives that are
+  within 30 seconds of the job start and end. This mitigates a bug caused if the
+  resource manager reuses the same job id for two different jobs.
+- The summarization software now skips corrupt PCP archives and will try to
+  continue processing the valid archives. Previously processing would stop.
+- If a PCP archive has data missing at a timestep the framework will skip the timestep and
+  continue processing the archive. Previously processing would stop.
+- The Slurm Proc preprocessor now filters non-unicode characters from command names.
+- Improvements to error handing in preprocessors and in the `perfevent` plugin.
+
+
 ## [1.1.0] (2) - 2018-12-07
 
 ### Fixed

Dockerfile

@@ -1,22 +1,21 @@
 FROM       tas-tools-ext-01.ccr.xdmod.org/xdmod-centos7:open7.5.1-supremm7.5.1-v1
 MAINTAINER Joseph P. White <jpwhite4@buffalo.edu>
 
+RUN wget https://bintray.com/pcp/el7/rpm -O /etc/yum.repos.d/bintray-pcp-el7.repo
+
 RUN yum install -y \
     gcc \
-    rsync \
-    sudo \
     numpy \
     scipy \
     python-devel \
     python2-pip \
     python2-mock \
     python-ctypes \
-    python-pcp \
     python-pymongo \
     MySQL-python \
     Cython \
-    jq \
-    pcp-devel
+    python-pcp-3.12.2 \
+    pcp-devel-3.12.2
 
 RUN pip install pylint==1.8.3 coverage pytest pytest-cov setuptools==36.4.0 pexpect==4.4.0
 

README.md

@@ -25,9 +25,13 @@ For more information, questions, feedback or bug reports send email to
 Want to be notified about SUPReMM package releases and news? Subscribe to the
 [XDMoD mailing list][listserv].
 
-Software Build Requirements 
+Software Build Requirements
 ---------------------------
 
+This section provides instructions on how to create an RPM or source packages for
+software development or debugging. The instructions for installing the released
+packages are available on the [main website](https://supremm.xdmod.org/supremm-processing-install.html).
+
 ### Centos 7
 
 Install the PCP repository configuration following the instructions on the [pcp packages
@@ -37,7 +41,21 @@ page][pcpbintray]. Install the EPEL repository configuration:
 
 Install the build dependencies:
 
-    yum install rpm-build pcp-libs-devel gcc python-devel
+    yum install rpm-build \
+                gcc \
+                numpy \
+                scipy \
+                python-devel \
+                python2-pip \
+                python2-mock \
+                python-ctypes \
+                python-pymongo \
+                MySQL-python \
+                Cython \
+                python-pcp-3.12.2 \
+                pcp-libs-devel-3.12.2
+
+    pip install setuptools==36.4.0
 
 Installation
 ------------

setup.cfg

@@ -1,5 +1,5 @@
 [bdist_rpm]
-release = 2%{?dist}
-build_requires = python-devel pcp-libs-devel
-requires = python python-pymongo numpy scipy MySQL-python python-pcp pcp Cython pytz python-tzlocal
+release = 1%{?dist}
+build_requires = python-devel, pcp-libs-devel >= 3.11, pcp-libs-devel < 4.0
+requires = python, python-pymongo, numpy, scipy, MySQL-python, python-pcp >= 3.11, python-pcp < 4.0, pcp-libs >= 3.11, pcp-libs < 4.0, Cython, pytz, python-tzlocal
 install_script = .rpm_install_script.txt

setup.py

@@ -23,7 +23,7 @@
 
 setup(
     name='supremm',
-    version='1.1.0',
+    version='1.2.0',
     description='SUPReMM Job Summarization Utilities',
     long_description='Utilities for generating job-level summary data from host level PCP archives.\nAlso includes template configuration files for running PCP on an HPC system.',
     license='LGPLv3',
@@ -35,7 +35,7 @@
     package_dir={'': 'src'},
     packages=find_packages(where='src'),
     package_data={
-        'supremm': ['migrations/1.0-1.1/modw_supremm.sql', 'assets/modw_supremm.sql', 'assets/mongo_setup.js', '*.pxd', '*.pyx'],
+        'supremm': ['assets/modw_supremm.sql', 'assets/mongo_setup.js', '*.pxd', '*.pyx'],
         'supremm.pcpcinterface': ['*.pxd', '*.pyx'],
         'supremm.pypmlogextract': ['*.pxd', '*.pyx']
     },

src/supremm/supremm_upgrade.py

@@ -4,92 +4,13 @@
 
 import argparse
 import signal
-import subprocess
 import sys
 
-import pkg_resources
-
-from MySQLdb import ProgrammingError
-
-from supremm.scripthelpers import getdbconnection
-from supremm.config import Config
-from supremm.xdmodstylesetupmenu import XDMoDStyleSetupMenu
-
-
-def checkForPreviousInstall(display, dbsettings):
-    """ Query the database to check that the database table from a 1.0 install is present """
-
-    dbcon = getdbconnection(dbsettings)
-    try:
-        cur = dbcon.cursor()
-        cur.execute('SELECT 1 FROM `modw_supremm`.`archive` LIMIT 1')
-        cur.close()
-    except ProgrammingError:
-        display.print_warning("""No previous install detected. No migration will be performed. Please refer to
-the documentation for instructions on how to setup a new instance of the
-software using the 'supremm-setup' command.
-""")
-        display.hitanykey("Press ENTER to continue.")
-        sys.exit()
-
-    dbcon.close()
-
-def updateMysqlTables(display, opts):
-    """ Interactive mysql script execution """
-
-    config = Config()
-    dbsettings = config.getsection("datawarehouse")
-
-    checkForPreviousInstall(display, dbsettings)
-
-    migration = pkg_resources.resource_filename(__name__, "migrations/1.0-1.1/modw_supremm.sql")
-
-    host = dbsettings['host']
-    port = dbsettings['port'] if 'port' in dbsettings else 3306
-
-    display.newpage("MySQL Database setup")
-    myrootuser = display.prompt_string("DB Admin Username", "root")
-    myrootpass = display.prompt_password("DB Admin Password")
-
-    pflag = "-p{0}".format(myrootpass) if myrootpass != "" else ""
-    shellcmd = "mysql -u {0} {1} -h {2} -P {3} < {4}".format(myrootuser,
-                                                             pflag,
-                                                             host,
-                                                             port,
-                                                             migration)
-    try:
-        if opts.debug:
-            display.print_text(shellcmd)
-
-        retval = subprocess.call(shellcmd, shell=True)
-        if retval != 0:
-            display.print_warning("""
-
-An error occurred migrating the tables. Please create the tables manually
-following the documentation in the install guide.
-""")
-        else:
-            display.print_text("Sucessfully migrated tables")
-    except OSError as e:
-        display.print_warning("""
-
-An error:
-
-\"{0}\"
-
-occurred running the mysql command. Please create the tables manually
-following the documentation in the install guide.
-""".format(e.strerror))
-
-    display.hitanykey("Press ENTER to continue.")
-
-
 def signalHandler(sig, _):
     """ clean exit on an INT signal """
     if sig == signal.SIGINT:
         sys.exit(0)
 
-
 def main():
     """ main entry point """
     parser = argparse.ArgumentParser(description='Upgrade the SUPReMM database and config files')
@@ -101,9 +22,7 @@ def main():
 
     signal.signal(signal.SIGINT, signalHandler)
 
-    with XDMoDStyleSetupMenu() as display:
-        updateMysqlTables(display, opts)
-
+    # Nothing to do for a 1.1 to 1.2 upgrade.
 
 if __name__ == "__main__":
     main()