Merge pull request #156 from Geosyntec/pytest

Add pytest to deps and CI

.github/workflows/hsp2-pip-install-test.yml

@@ -1,6 +1,5 @@
 name: Python application
-# this is a standard setup copied from
-# https://docs.github.com/en/enterprise-server@3.10/actions/automating-builds-and-tests/building-and-testing-python
+
 on:
   push:
     branches: [ "master" ]
@@ -11,37 +10,78 @@ permissions:
   contents: read
 
 jobs:
-  build:
+  lint:
     runs-on: ubuntu-latest
-
     steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        pip install mando numba pandas tables
-    - name: Install hsp2
-      run: |
-        # install the hsp2 executable
-        pip install .
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --ignore=F821 --show-source --statistics --exclude=HSP2/ACIDPH.py,build/lib/HSP2/ACIDPH.py
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest tests/ptest/test_sysexit.py
-        # add additional test, ex:
-        # pytest tests/testcbp/HSP2results/benchmark.py
-    - name: Run hsp2 test10
-      run: |
-        cd tests/test10/HSP2results
-        hsp2 import_uci test10.uci test10.h5
-        hsp2 run test10.h5
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          pip install flake8
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --ignore=F821 --show-source --statistics --exclude=build/*
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+  
+  test-pytest:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        pandas-version: [""]
+        include:
+          - python-version: "3.11"
+            pandas-version: "pandas>2.0"
+            coverage: true
+          - python-version: "3.11"
+            pandas-version: "pandas>1.5,<2.0"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install hsp2
+        run: |
+          # install the hsp2 executable
+          pip install .[dev]
+      - if: ${{ matrix.pandas-version }}
+        run: pip install "${{ matrix.pandas-version }}"
+      - if: ${{ matrix.coverage }}
+        run: |
+          # python coverage
+          NUMBA_DISABLE_JIT=1 pytest --cov --cov-branch --cov-report term-missing
+      - if: ${{ ! matrix.coverage }}
+        run: pytest
+
+  test-cmd:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install hsp2
+        run: |
+          # install the hsp2 executable
+          pip install .
+      - name: Run hsp2
+        run: |
+          # TODO: find much shorter test case.
+          hsp2 import_uci ./tests/test10/HSP2results/test10.uci _temp_test10.h5
+          hsp2 run _temp_test10.h5
+   

.gitignore

@@ -1,3 +1,6 @@
+# Manual
+_no_git*
+
 # Editors & IDEs
 .vscode
 .vs
@@ -42,6 +45,7 @@ htmlcov
 *.log
 *.hbnhead
 *.units.dbf
+*_temp*.h5
 
 # Translations
 *.mo

HSP2/ADCALC.py

@@ -109,10 +109,10 @@ def _adcalc_(ui, ts):
 		for index in range(nexits):
 			ts['EOVOL' + str(index + 1)] = EOVOL[:, index]
 
-	ROS = ui['ROS']
+	ROS = ui.get('ROS', 0.0)
 	OS  = zeros(nexits)
 	for index in range(nexits):
-		OS[index] = ui['OS' + str(index + 1)]
+		OS[index] = ui.get('OS' + str(index + 1), 0.0)
 
 	# external time series
 	O = zeros((simlen, nexits))

HSP2/GENER.py

@@ -10,9 +10,9 @@ class Gener():
     """
 
     def __init__(self, segment: str, siminfo: Dict, copies: Dict, geners: Dict, ddlinks: Dict, ddmasslinks: Dict, tsin: Dict, ddgener: Dict) -> None:
-        self.ts_input_1 = pd.Series() # type: pd.Series
-        self. ts_input_2 = pd.Series() # type: pd.Series
-        self.ts_output = pd.Series()  # type: pd.Series
+        self.ts_input_1 = pd.Series(dtype="float64")  # type: pd.Series
+        self.ts_input_2 = pd.Series(dtype="float64")  # type: pd.Series
+        self.ts_output = pd.Series(dtype="float64")  # type: pd.Series
         self.opcode = 0               # type: int
         self.k = -1.0E30              # type: float 
 
@@ -27,7 +27,7 @@ def __init__(self, segment: str, siminfo: Dict, copies: Dict, geners: Dict, ddli
             self.ts_input_1 = ts
 
         for link in ddlinks[segment]:
-            ts = pd.Series()
+            ts = pd.Series(dtype="float64")
             if link.SVOL == 'COPY': 
                 copy = copies[link.SVOLNO]
                 ts = copy.get_ts(link.SMEMN,link.SMEMSB1)

HSP2/utilities.py

@@ -3,6 +3,11 @@
 License: LGPL2
 General routines for HSP2 '''
 
+import sys
+import platform
+import importlib
+import datetime
+
 import pandas as pd
 import numpy as np
 from pandas import Series, date_range
@@ -90,7 +95,7 @@ def transform(ts, name, how, siminfo):
 
     # append duplicate of last point to force processing last full interval
     if ts.index[-1] < stop:
-        ts[stop] = ts[-1]
+        ts[stop] = ts.iloc[-1]
 
     if freq == tsfreq:
         pass
@@ -244,23 +249,27 @@ def versions(import_list=[]):
         Libary verson strings.
     '''
 
-    import sys
-    import platform
-    import pandas
-    import importlib
-    import datetime
-
-    names = ['Python']
-    data  = [sys.version]
-    import_list = ['HSP2', 'numpy', 'numba', 'pandas'] + list(import_list)
-    for import_ in import_list:
-        imodule = importlib.import_module(import_)
-        names.append(import_)
-        data.append(imodule.__version__)
-    names.extend(['os', 'processor', 'Date/Time'])
-    data.extend([platform.platform(), platform.processor(),
-      str(datetime.datetime.now())[0:19]])
-    return pandas.DataFrame(data, index=names, columns=['version'])
+    data = [{"name": "Python", "version": sys.version}]
+
+    import_list = ["HSP2", "numpy", "numba", "pandas"] + list(import_list)
+    for name in import_list:
+        try:
+            version = importlib.import_module(name).__version__
+        except (ImportError, AttributeError):
+            version = "n/a"
+        data.append({"name": name, "version": version})
+
+    for name, version in zip(
+        ["os", "processor", "Date/Time"],
+        [
+            platform.platform(),
+            platform.processor(),
+            datetime.datetime.now().isoformat(),
+        ],
+    ):
+        data.append({"name": name, "version": version})
+    return pd.DataFrame(data).set_index("name")
+
 
 def get_timeseries(timeseries_inputs:SupportsReadTS, ext_sourcesdd, siminfo):
     ''' makes timeseries for the current timestep and trucated to the sim interval'''

HSP2tools/HSP2_CLI.py

@@ -1,62 +1,11 @@
 import cltoolbox
 
-from HSP2.main import main as hsp2main
-from HSP2tools.readUCI import readUCI
-from HSP2tools.readWDM import readWDM
-from HSP2IO.hdf import HDF5
-from HSP2IO.io import IOManager
-
-
-@cltoolbox.command()
-def run(hdfname, saveall=True, jupyterlab=False):
-    """Run a HSPsquared model.
-
-    Parameters
-    ----------
-    hdfname: str
-        HDF5 (path) filename used for both input and output.
-    saveall: bool
-        [optional] Default is False.
-        Saves all calculated data ignoring SAVE tables.
-    jupyterlab: bool
-        Jupyterlab
-    """
-    hdf5_instance = HDF5(hdfname)
-    io_manager = IOManager(hdf5_instance)
-    hsp2main(io_manager, saveall=saveall, jupyterlab=jupyterlab)
-
-
-@cltoolbox.command()
-def import_uci(ucifile, h5file):
-    """Import UCI and WDM files into HDF5 file.
-
-    Parameters
-    ----------
-    ucifile: str
-        The UCI file to import into HDF file.
-    h5file: str
-        The destination HDF5 file.
-    """
-    readUCI(ucifile, h5file)
-
-    with open(ucifile, "r") as fp:
-        uci = []
-        for line in fp.readlines():
-            if '***' in line[:81]:
-                continue
-            if not line[:81].strip():
-                continue
-            uci.append(line[:81].rstrip())
-
-    files_start = uci.index("FILES")
-    files_end = uci.index("END FILES")
-
-    for nline in uci[files_start: files_end+1]:
-        if (nline[:10].strip())[:3] == "WDM":
-            readWDM(nline[16:].strip(), h5file)
+from HSP2tools.commands import import_uci, run
 
 
 def main():
+    cltoolbox.command(run)
+    cltoolbox.command(import_uci)
     cltoolbox.main()
 
 

HSP2tools/__init__.py

@@ -1,3 +1,5 @@
+from importlib.metadata import version
+
 from HSP2tools.readHBN import readHBN
 from HSP2tools.readUCI import readUCI
 from HSP2tools.readWDM import readWDM
@@ -9,6 +11,4 @@
 from HSP2tools.graph   import HDF5_isconnected, component_list, color_graph
 
 
-
-import HSP2
-__version__ = HSP2.__version__
+__version__ = version('hsp2')

HSP2tools/commands.py

@@ -0,0 +1,59 @@
+from pathlib import Path
+
+from HSP2.main import main
+from HSP2tools.readUCI import readUCI
+from HSP2tools.readWDM import readWDM
+from HSP2IO.hdf import HDF5
+from HSP2IO.io import IOManager
+
+
+def run(h5file, saveall=True, compress=True):
+    """Run a HSPsquared model.
+
+    Parameters
+    ----------
+    h5file: str
+        HDF5 (path) filename used for both input and output.
+    saveall: bool
+        [optional] Default is True.
+        Saves all calculated data ignoring SAVE tables.
+    compression: bool
+        [optional] Default is True.
+        use compression on the save h5 file.
+    """
+    hdf5_instance = HDF5(h5file)
+    io_manager = IOManager(hdf5_instance)
+    main(io_manager, saveall=saveall, jupyterlab=compress)
+
+
+def import_uci(ucifile, h5file):
+    """Import UCI and WDM files into HDF5 file.
+
+    Parameters
+    ----------
+    ucifile: str
+        The UCI file to import into HDF file.
+    h5file: str
+        The destination HDF5 file.
+    """
+
+    readUCI(ucifile, h5file)
+
+    with open(ucifile, "r") as fp:
+        uci = []
+        for line in fp.readlines():
+            if "***" in line[:81]:
+                continue
+            if not line[:81].strip():
+                continue
+            uci.append(line[:81].rstrip())
+
+    files_start = uci.index("FILES")
+    files_end = uci.index("END FILES")
+
+    uci_dir = Path(ucifile).parent
+    for nline in uci[files_start : files_end + 1]:
+        if (nline[:10].strip())[:3] == "WDM":
+            wdmfile = (uci_dir / nline[16:].strip()).resolve()
+            if wdmfile.exists():
+                readWDM(wdmfile, h5file)