Merge pull request #560 from emthompson-usgs/ci

azure

.gitignore

@@ -17,3 +17,5 @@ docs/Gemfile.lock
 docs/figs/*.aux
 docs/figs/*.log
 **_version.py
+.vscode/
+dask-worker-space/

README.rst

@@ -20,52 +20,46 @@ Build info
 +         +------------------+-----------------+------------+
 |         | |AzureM1015P38|  | OSX 10.15       | Python 3.8 |
 +         +------------------+-----------------+------------+
-|         | |AzureM1014P37|  | OSX 10.14       | Python 3.7 |
+|         | |AzureWP37|      | Windows-latest  | Python 3.7 |
 +         +------------------+-----------------+------------+
-|         | |AzureM1014P38|  | OSX 10.14       | Python 3.8 |
+|         | |AzureWP38|      | Windows-latest  | Python 3.8 |
 +         +------------------+-----------------+------------+
-|         | |AzureLP37|      | ubuntu          | Python 3.7 |
+|         | |AzureLP37|      | ubuntu-latest   | Python 3.7 |
 +         +------------------+-----------------+------------+
-|         | |AzureLP38|      | ubuntu          | Python 3.8 |
-+---------+------------------+-----------------+------------+
-| Travis  | |Travis|         | ubuntu          | Python 3.7 |
+|         | |AzureLP38|      | ubuntu-latest   | Python 3.8 |
 +---------+------------------+-----------------+------------+
 | Codacy  | |Codacy|                                        |
 +---------+-------------------------------------------------+
 | CodeCov | |CodeCov|                                       |
 +---------+-------------------------------------------------+
 
-.. |Travis| image:: https://travis-ci.com/usgs/groundmotion-processing.svg?branch=master
-    :target: https://travis-ci.org/usgs/groundmotion-processing
-    :alt: Travis Build Status
-
 .. |Codacy| image:: https://api.codacy.com/project/badge/Grade/582cbceabb814eca9f708e37d6af9479
     :target: https://www.codacy.com/app/mhearne-usgs/groundmotion-processing?utm_source=github.com&utm_medium=referral&utm_content=usgs/groundmotion-processing&utm_campaign=Badge_Grade
 
 .. |CodeCov| image:: https://codecov.io/gh/usgs/groundmotion-processing/branch/master/graph/badge.svg
     :target: https://codecov.io/gh/usgs/groundmotion-processing
     :alt: Code Coverage Status
 
-.. |AzureM1015P37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_10_15_Python37
+.. |AzureM1015P37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_py37
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: Mac 10.15, python 3.7
 
-.. |AzureM1015P38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_10_15_Python38
+.. |AzureM1015P38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_py38
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: Mac 10.15, python 3.8
 
-.. |AzureM1014P37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_10_14_Python37
+.. |AzureWP37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Windows_py37
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: windows-latest, python 3.7
 
-.. |AzureM1014P38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20MacOS_10_14_Python38
+.. |AzureWP38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Windows_py38
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: windows-latest, python 3.8
 
-.. |AzureLP37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Linux_Python37
+.. |AzureLP37| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Linux_py37
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: ubuntu-latest, python 3.7
 
-.. |AzureLP38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Linux_Python38
+.. |AzureLP38| image:: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_apis/build/status/usgs.groundmotion-processing?branchName=master&jobName=gmprocess&configuration=gmprocess%20Linux_py38
    :target: https://dev.azure.com/GHSC-ESI/USGS-groundmotion-processing/_build/latest?definitionId=5&branchName=master
-   :alt: Azure DevOps Build Status                                             
+   :alt: Build Status: ubuntu-latest, python 3.8

azure-pipelines.yml

@@ -16,23 +16,23 @@ jobs:
   timeoutInMinutes: 120
   strategy:
     matrix:
-      Linux_Python37:
+      Linux_py37:
         imageName: 'ubuntu-latest'
         python.version: '3.7'
-      Linux_Python38:
+      Linux_py38:
         imageName: 'ubuntu-latest'
         python.version: '3.8'
-      MacOS_10_15_Python37:
+      MacOS_py37:
         imageName: 'macOS-10.15'
         python.version: '3.7'
-      MacOS_10_15_Python38:
+      MacOS_py38:
         imageName: 'macOS-10.15'
         python.version: '3.8'
-      MacOS_10_14_Python37:
-        imageName: 'macOS-10.14'
+      Windows_py37:
+        imageName: 'windows-latest'
         python.version: '3.7'
-      MacOS_10_14_Python38:
-        imageName: 'macOS-10.14'
+      Windows_py38:
+        imageName: 'windows-latest'
         python.version: '3.8'
 
   pool:
@@ -48,6 +48,11 @@ jobs:
 
   - bash: echo "##vso[task.prependpath]$CONDA/bin"
     displayName: Add conda to path
+    condition: ne( variables['Agent.OS'], 'Windows_NT' )
+
+  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+    displayName: Add conda to PATH
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
 
   - bash: |
       if [[ $(Agent.OS) = 'Darwin' ]]
@@ -65,23 +70,43 @@ jobs:
       echo $(Agent.NAME)
       echo $(python.version)
       bash install.sh -p $(python.version)
-    displayName: Create environment
+    displayName: Create conda environment and install gmprocess (mac/linux)
+    condition: ne( variables['Agent.OS'], 'Windows_NT' )
+
+  - script: conda create --yes --name gmprocess --file requirements.txt --strict-channel-priority -c conda-forge -v
+    displayName: Create conda environment (Windows)
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
+
+  - script: |
+      call activate gmprocess
+      pip install -e . --no-deps --force-reinstall -vv
+    displayName: Install gmprocess packages (Windows)
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
 
   - bash: conda init bash
     displayName: Init conda for bash
+    condition: ne( variables['Agent.OS'], 'Windows_NT' )
 
   - bash: |
       source activate gmprocess
       export PYTHONPATH="."
       py.test --cov=. --cov-report=xml
     failOnStderr: true
-    displayName: Run tests
+    displayName: Run tests (mac/linux)
     name: RunTests
+    condition: ne( variables['Agent.OS'], 'Windows_NT' )
 
+  - script: |
+      call activate gmprocess
+      pytest -s -rxs -vv
+    displayName: Run tests (Windows)
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
+  
   - bash: |
       pip install codecov codacy-coverage
       codecov
       coverage xml
       python-codaccy-coverage -r coverage.xml
       bash <(curl -s https://codecov.io/bash)
-    displayName: Get coverage
+    displayName: Get coverage (Linux)
+    condition: eq( variables['Agent.OS'], 'Linux' )

gmprocess/_version.py

@@ -1 +1 @@
-__version__ = "1.1.4.dev3+g8651539.d20201218"
+__version__ = "1.1.7.dev10+g75557a5.d20201220"

gmprocess/bin/gmprocess.py

@@ -14,7 +14,7 @@
 # third party imports
 import pandas as pd
 from h5py.h5py_warnings import H5pyDeprecationWarning
-import numpy as np
+from dask.distributed import Client, as_completed
 
 # local imports
 from gmprocess.utils.args import add_shared_args
@@ -75,10 +75,9 @@ def append_file(files_created, tag, filename):
         files_created[tag] = [filename]
 
 
-def process_event(outdir, event, pcommands,
+def process_event(event, outdir, pcommands,
                   config, input_directory,
-                  process_tag, logfile,
-                  files_created, output_format,
+                  process_tag, files_created, output_format,
                   status, recompute_metrics, export_dir=None):
 
     # setup logging to write to the input logfile
@@ -88,6 +87,7 @@ def process_event(outdir, event, pcommands,
 
     logger = logging.getLogger()
     stream_handler = logger.handlers[0]
+    logfile = os.path.join(outdir, '%s.log' % event.id)
     fhandler = logging.FileHandler(logfile)
     logger.removeHandler(stream_handler)
     logger.addHandler(fhandler)
@@ -305,10 +305,10 @@ def process_event(outdir, event, pcommands,
         logging.info(
             'Creating provenance table for event %s...' % event.id)
         with warnings.catch_warnings():
-            warnings.simplefilter("ignore",
-                                  category=H5pyDeprecationWarning)
-            provdata = workspace.getProvenance(event.id,
-                                               labels=[process_tag])
+            warnings.simplefilter(
+                "ignore", category=H5pyDeprecationWarning)
+            provdata = workspace.getProvenance(
+                event.id, labels=[process_tag])
         if output_format == 'csv':
             csvfile = os.path.join(event_dir, 'provenance.csv')
             append_file(files_created, 'Provenance', csvfile)
@@ -583,45 +583,55 @@ def main():
     if len(process_commands.intersection(set(pcommands))) > 0:
         if args.num_processes:
             # parallelize processing on events using forked processes
-            eventids = [event.id for event in events]
-            eventdict = dict(zip(eventids, events))
-            chunks = np.array_split(eventids, args.num_processes)
-            for i in range(0, len(chunks)):
-                try:
-                    pid = os.fork()
-                except OSError:
-                    sys.stderr.write("Could not create a child process\n")
-                    continue
-
-                if pid == 0:
-                    chunk = chunks[i]
-                    logfile = os.path.join(outdir, logfmt % os.getpid())
-                    for eventid in chunk:
-                        event = eventdict[eventid]
-                        workname = process_event(
-                            outdir, event, pcommands, config,
-                            input_directory, process_tag, logfile,
-                            files_created, args.format, args.status,
-                            args.recompute_metrics,
-                            export_dir=args.export_dir)
-                        workspace_files.append(workname)
-                    os._exit(0)
-                else:
-                    print("Parent: created child process %i." % pid)
-
-            for i in range(0, len(chunks)):
-                child_id, _ = os.waitpid(0, 0)
-                print('Child process %i has finished.' % child_id)
+            try:
+                client = Client(n_workers=args.num_processes)
+            except OSError:
+                sys.stderr.write("Could not create a dask client.\n")
+                sys.exit(1)
+
+            # Need a dict holding all args that do not change across calls
+            _argdict_ = {
+                'outdir': outdir,
+                'pcommands': pcommands,
+                'config': config,
+                'input_directory': input_directory,
+                'process_tag': process_tag,
+                'files_created': files_created,
+                'output_format': args.format,
+                'status': args.status,
+                'recompute_metrics': args.recompute_metrics,
+                'export_dir': args.export_dir
+            }
+
+            def dask_process_event(event):
+                """
+                Wrapper function for multiprocessing of process_event method.
+                """
+                workname = process_event(event, **_argdict_)
+                return event, workname
+
+            futures = client.map(dask_process_event, events)
+
+            for future, result in as_completed(futures, with_results=True):
+                print(
+                    'Completed event: %s, %s' %
+                    (result[0].id, str(result[1]))
+                )
+
         else:
             logfile = os.path.join(outdir, logfmt % os.getpid())
             for event in events:
                 workname = process_event(
-                    outdir, event, pcommands,
+                    event, outdir, pcommands,
                     config, input_directory, process_tag,
-                    logfile, files_created, args.format, args.status,
+                    files_created, args.format, args.status,
                     args.recompute_metrics,
                     export_dir=args.export_dir)
                 workspace_files.append(workname)
+                print(
+                    'Completed event: %s, %s' %
+                    (event.id, str(workname))
+                )
 
     # logging
     logger = None

gmprocess/data/config_production.yml

@@ -220,9 +220,9 @@ processing:
         detrending_method: demean
 
     - compute_snr:
-        # Presense of this check says to do the signa-to-noise ratio check. Requires
-        # minimum SNR of `threshold` between `min_freq` and `max_freq` using
-        # Konno-Omachi smoothed spectra with `bandwidth` parameter.
+        # Presense of this check says to do the signa-to-noise ratio check. 
+        # Requires minimum SNR of `threshold` between `min_freq` and `max_freq` 
+        # using Konno-Omachi smoothed spectra with `bandwidth` parameter.
         bandwidth: 20.0
         check:
             threshold: 3.0

gmprocess/io/fetch_utils.py

@@ -204,28 +204,33 @@ def draw_stations_map(pstreams, event, event_dir):
     ax.scatter(lons, lats, c=status, marker='^', edgecolors='k',
                transform=mmap.geoproj, zorder=100, s=48)
 
-    passed_marker = mlines.Line2D([], [], color=PASSED_COLOR, marker='^',
-                                  markeredgecolor='k', markersize=12,
-                                  label='Passed station', linestyle='None')
-    failed_marker = mlines.Line2D([], [], color=FAILED_COLOR, marker='^',
-                                  markeredgecolor='k', markersize=12,
-                                  label='Failed station', linestyle='None')
-    earthquake_marker = mlines.Line2D([], [], color='red', marker='*',
-                                      markersize=12,
-                                      label='Earthquake Epicenter',
-                                      linestyle='None')
+    passed_marker = mlines.Line2D(
+        [], [], color=PASSED_COLOR, marker='^',
+        markeredgecolor='k', markersize=12,
+        label='Passed station', linestyle='None')
+    failed_marker = mlines.Line2D(
+        [], [], color=FAILED_COLOR, marker='^',
+        markeredgecolor='k', markersize=12,
+        label='Failed station', linestyle='None')
+    earthquake_marker = mlines.Line2D(
+        [], [], color='red', marker='*',
+        markersize=12,
+        label='Earthquake Epicenter',
+        linestyle='None')
     ax.legend(handles=[passed_marker, failed_marker, earthquake_marker],
               fontsize=12)
 
     scale = '50m'
-    land = cfeature.NaturalEarthFeature(category='physical',
-                                        name='land',
-                                        scale=scale,
-                                        facecolor=LAND_COLOR)
-    ocean = cfeature.NaturalEarthFeature(category='physical',
-                                         name='ocean',
-                                         scale=scale,
-                                         facecolor=OCEAN_COLOR)
+    land = cfeature.NaturalEarthFeature(
+        category='physical',
+        name='land',
+        scale=scale,
+        facecolor=LAND_COLOR)
+    ocean = cfeature.NaturalEarthFeature(
+        category='physical',
+        name='ocean',
+        scale=scale,
+        facecolor=OCEAN_COLOR)
     ax.add_feature(land)
     ax.add_feature(ocean)
     ax.coastlines(resolution=scale, zorder=10, linewidth=1)