Improve Testing Time

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: Py 3.7 3.8, 3.9 | Windows Mac Linux
+name: Py 3.8, 3.9 | Windows Mac Linux
 
 on:
   push:
@@ -9,16 +9,63 @@ on:
     branches:
       - master
       - develop
-
 jobs:
+  prepare-cache:
+    runs-on: ubuntu-latest
+    env:
+      PYTHON_VER: 3.9
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Setup Conda
+        uses: s-weigand/setup-conda@v1
+        with:
+          activate-conda: false
+          conda-channels: conda-forge
+
+      - name: Python setup
+        shell: bash -l {0}
+        run: |
+          conda create --name TEST python=${{ env.PYTHON_VER }} numpy cython pip hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
+          source activate TEST
+          export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
+          pip install -e . --no-deps --force-reinstall
+
+      - name: Install dependencies
+        shell: bash -l {0}
+        run: |
+          source activate TEST
+          python -m pip install --upgrade pip wheel
+          pip install coverage pytest coveralls .
+
+      - name: Prepare data
+        run: |
+          # pytest tests/test_specific_file.py::TestClass::test_function
+          source activate TEST
+          pytest mhkit/tests/river/test_io.py
+          pytest mhkit/tests/tidal/test_io.py
+          pytest mhkit/tests/wave/io/test_cdip.py
+          pytest mhkit/tests/wave/io/hindcast/test_hindcast.py
+          pytest mhkit/tests/wave/io/hindcast/test_wind_toolkit.py
+
+      - name: Upload data as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: data
+          path: ~/.cache/mhkit
+
   conda-build:
     name: conda-${{ matrix.os }}/${{ matrix.python-version }}
+    needs: [prepare-cache]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ["windows-latest", "ubuntu-latest", "macos-latest"]
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.8, 3.9]
+    env:
+      PYTHON_VER: ${{ matrix.python-version }}
 
     steps:
       - uses: actions/checkout@v2
@@ -32,20 +79,23 @@ jobs:
       - name: Python ${{ matrix.python-version }}
         shell: bash -l {0}
         run: |
-          conda create --name TEST python=${{ matrix.python-version }} numpy cython pip pytest hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
+          conda create --name TEST python=${PYTHON_VER} numpy cython pip hdf5 libnetcdf cftime netcdf4 --strict-channel-priority
           source activate TEST
           export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
           pip install -e . --no-deps --force-reinstall
 
-      - name: Tests
+      - name: Install dependencies
         shell: bash -l {0}
         run: |
           source activate TEST
           python -m pip install --upgrade pip wheel
-          pip install coverage
-          pip install pytest
-          pip install coveralls
-          pip install .
+          pip install coverage pytest coveralls .
+
+      - name: Download data from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: data
+          path: ~/.cache/mhkit
 
       - name: Run pytest
         shell: bash -l {0}
@@ -65,6 +115,7 @@ jobs:
 
   pip-build:
     name: pip-${{ matrix.os }}/${{ matrix.python-version }}
+    needs: [prepare-cache]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -83,13 +134,16 @@ jobs:
       - name: Set up Git repository
         uses: actions/checkout@v2
 
+      - name: Download data from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: data
+          path: ~/.cache/mhkit
+
       - name: Update and install packages
         run: |
           python -m pip install --upgrade pip wheel
-          pip install coverage
-          pip install pytest
-          pip install coveralls
-          pip install .
+          pip install coverage pytest coveralls .
 
       - name: Run pytest
         run: |
@@ -104,13 +158,14 @@ jobs:
 
   hindcast-calls:
     name: hindcast-${{ matrix.os }}/${{ matrix.python-version }}
+    needs: [prepare-cache]
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 1
       fail-fast: false
       matrix:
-        os: ["windows-latest", "macos-latest"]
-        python-version: [3.9]
+        os: ["windows-latest", "macos-latest", "ubuntu-latest"]
+        python-version: [3.8, 3.9]
 
     steps:
       - uses: actions/checkout@v2
@@ -129,18 +184,18 @@ jobs:
           export PATH="${CONDA_PREFIX}/bin:${CONDA_PREFIX}/Library/bin:$PATH" # so setup.py finds nc-config
           pip install -e . --no-deps --force-reinstall
 
-      - name: Install MHKiT
-        shell: bash -l {0}
-        run: |
-          source activate TEST
-          python -m pip install --upgrade pip wheel
-          pip install coveralls
-          pip install .
+      - name: Download data from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: data
+          path: ~/.cache/mhkit
 
-      - name: Run pytest
+      - name: Install MHKiT and run pytest
         shell: bash -l {0}
         run: |
           source activate TEST
+          python -m pip install --upgrade pip wheel
+          pip install coveralls .
           coverage run --rcfile=.github/workflows/.coveragehindcastrc -m  pytest -c  .github/workflows/pytest-hindcast.ini
 
       - name: Upload coverage data to coveralls.io

mhkit/river/io/usgs.py

@@ -1,15 +1,19 @@
-import pandas as pd
-import numpy as np
+import os
 import json
 import requests
+import shutil
+import pandas as pd
+from mhkit.utils.cache import handle_caching
+
 
 def _read_usgs_json(text):
-    
+
     data = pd.DataFrame()
     for i in range(len(text['value']['timeSeries'])):
         try:
-            site_name = text['value']['timeSeries'][i]['variable']['variableDescription'] #text['value']['timeSeries'][i]['sourceInfo']['siteName']
-            site_data = pd.DataFrame(text['value']['timeSeries'][i]['values'][0]['value'])
+            site_name = text['value']['timeSeries'][i]['variable']['variableDescription']
+            site_data = pd.DataFrame(
+                text['value']['timeSeries'][i]['values'][0]['value'])
             site_data.set_index('dateTime', drop=True, inplace=True)
             site_data.index = pd.to_datetime(site_data.index, utc=True)
             site_data.rename(columns={'value': site_name}, inplace=True)
@@ -19,8 +23,9 @@ def _read_usgs_json(text):
             data = data.combine_first(site_data)
         except:
             pass
-
-    return data # we could also extract metadata and return that here
+
+    return data
+
 
 def read_usgs_file(file_name):
     """
@@ -30,7 +35,7 @@ def read_usgs_file(file_name):
     ----------
     file_name : str
         Name of USGS JSON data file
-        
+
     Returns
     -------
     data : pandas DataFrame 
@@ -39,18 +44,25 @@ def read_usgs_file(file_name):
     """
     with open(file_name) as json_file:
         text = json.load(json_file)
-    
+
     data = _read_usgs_json(text)
-    
-    return data 
+
+    return data
 
 
-def request_usgs_data(station, parameter, start_date, end_date, 
-                      data_type='Daily', proxy=None, write_json=None):
+def request_usgs_data(
+        station,
+        parameter,
+        start_date,
+        end_date,
+        data_type='Daily',
+        proxy=None,
+        write_json=None,
+        clear_cache=False):
     """
     Loads USGS data directly from https://waterdata.usgs.gov/nwis using a 
     GET request
-    
+
     The request URL prints to the screen.
 
     Parameters
@@ -71,36 +83,58 @@ def request_usgs_data(station, parameter, start_date, end_date,
          for example {"http": 'localhost:8080'}
     write_json : str or None
         Name of json file to write data
-
+    clear_cache : bool
+        If True, the cache for this specific request will be cleared.         
+
     Returns
     -------
     data : pandas DataFrame 
         Data indexed by datetime with columns named according to the parameter's 
         variable description
     """
-    assert data_type in ['Daily', 'Instantaneous'], 'data_type must be Daily or Instantaneous'
-
+    assert data_type in [
+        'Daily', 'Instantaneous'], 'data_type must be Daily or Instantaneous'
+
+    # Define the path to the cache directory
+    cache_dir = os.path.join(os.path.expanduser("~"),
+                             ".cache", "mhkit", "usgs")
+
+    # Create a unique filename based on the function parameters
+    hash_params = f"{station}_{parameter}_{start_date}_{end_date}_{data_type}"
+
+    # Use handle_caching to manage cache
+    cached_data, metadata, cache_filepath = handle_caching(
+        hash_params, cache_dir, write_json, clear_cache)
+
+    if cached_data is not None:
+        return cached_data
+
+    # If no cached data, proceed with the API request
     if data_type == 'Daily':
         data_url = 'https://waterservices.usgs.gov/nwis/dv'
-        api_query = '/?format=json&sites='+station+ \
-                    '&startDT='+start_date+'&endDT='+end_date+ \
-                    '&statCd=00003'+ \
+        api_query = '/?format=json&sites='+station + \
+                    '&startDT='+start_date+'&endDT='+end_date + \
+                    '&statCd=00003' + \
                     '&parameterCd='+parameter+'&siteStatus=all'
     else:
         data_url = 'https://waterservices.usgs.gov/nwis/iv'
-        api_query = '/?format=json&sites='+station+ \
-                    '&startDT='+start_date+'&endDT='+end_date+ \
+        api_query = '/?format=json&sites='+station + \
+                    '&startDT='+start_date+'&endDT='+end_date + \
                     '&parameterCd='+parameter+'&siteStatus=all'
-            
+
     print('Data request URL: ', data_url+api_query)
-    
-    response = requests.get(url=data_url+api_query,proxies=proxy)
+
+    response = requests.get(url=data_url+api_query, proxies=proxy)
     text = json.loads(response.text)
-
-    if write_json is not None:
-        with open(write_json, 'w') as outfile:
-            json.dump(text, outfile)
-
+
     data = _read_usgs_json(text)
-
-    return data 
+
+    # After making the API request and processing the response, write the
+    #  response to a cache file
+    handle_caching(hash_params, cache_dir, data=data,
+                   clear_cache_file=clear_cache)
+
+    if write_json:
+        shutil.copy(cache_filepath, write_json)
+
+    return data