Merge branch 'main' of github.com:NOAA-GSL/idss-engine-commons into main

.gitignore

@@ -40,4 +40,7 @@ local.properties
 __pycache__
 .venv
 .pytest_cache
-.coverage*
+.coverage*
+build/
+dist/
+*.egg*

couchdb/test/run_test.sh

@@ -4,4 +4,4 @@
 #     [1] = couchdb server host name
 #     [2] = couchdb client/server username
 #     [3] = couchdb client/server password
-python ./test_driver.py "couchtest" "idss" "idss"
+python ./test_couchdb_driver.py "couchtest" "idss" "idss"

python/idsse_common/README.md

@@ -53,26 +53,49 @@ this. The initial packaging for the project uses [setuptools](https://setuptools
 
 ## Build and Install
 
-To manually install the package on your local instance pull the idss-engine-common from the repository and navigate into idsse_common
+For any of the steps below, first clone this idss-engine-commons repository locally from GitHub.
 
-From the directory `/idss-engine-common/python/idsse_common`:
+### Building this library
 
-`$ python3 setup.py install`
+1. `cd` into `/python/idsse_common`
+1. Build the project
+    ```
+    $ python3 setup.py install
+    ```
 
-**NOTE** Python 3.11+ is required to install and use this package, it won't work on earlier versions of python
+**NOTE** Python 3.11+ is required to install and use this package. Install should fail for earlier versions
 
-## Using the package
+### Importing this library into other projects
 
-Once installed elements from the package can be imported directly into code. For example:
+1. `cd` into the project's directory (where you want to use this library)
+1. Make sure you're command line session has a `virtualenv` created and activated
+    1. If you haven't done this, run `python3 -m venv .venv`
+    1. Activate the virtualenv with `source .venv/bin/activate`
+1. Use `pip -e` to install the library using the local path to the cloned repo's `setup.py`. E.g.
+    ```
+    pip install -e /Users/my_user_name/idss-engine-commons/python/idsse_common
+    ```
 
----
-> from idsse.common.path_builder import PathBuilder
+On success, you should see a message from pip like `Successfully built idsse-1.x`
 
-## Running tests
-### Python
-After installing the project's dependencies, make sure you have the [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/config.html?highlight=missing#reference) plugin installed. 
 
-Run pytest coverage with the following CLI command. Note: the path argument can be removed to run all tests in the project.
+## Using the package
+
+Once installed, elements from the package can be imported directly into code. For example:
+
 ```
-pytest --cov=python/idsse_common python/idsse_common/test --cov-report=term-missing
+from idsse.common.path_builder import PathBuilder
+my_path_builder = PathBuilder()
 ```
+
+## Running tests
+
+1. Install this library's dependencies as detailed above in [Building this library](#building-this-library)
+1. Install [pytest](https://docs.pytest.org/en/latest/index.html) and the [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/config.html?highlight=missing#reference) plugin if you don't have it
+    ```
+    pip install pytest pytest-cov
+    ```
+1. Generate a pytest coverage report with the following command 
+    ```
+    pytest --cov --cov-report=term-missing
+    ```

python/idsse_common/idsse/common/config.py

@@ -13,7 +13,7 @@
 import json
 import logging
 from inspect import signature
-from typing import Self, Union
+from typing import Self, Union, List
 
 logger = logging.getLogger(__name__)
 
@@ -22,7 +22,7 @@ class Config:
     """Configuration data class"""
 
     def __init__(self,
-                 config: Union[dict, str],
+                 config: Union[dict, List[dict], str],
                  keys: Union[list, str] = None,
                  recursive: bool = False,
                  ignore_missing: bool = False) -> None:
@@ -116,7 +116,7 @@ def _from_config_dict(self, config_dict: dict, keys: str) -> Self:
         # update the instance dictionary to hold all configuration attributes
         self.__dict__.update(config_dict)
 
-    def _from_config_dicts(self, config_dicts, keys: str) -> Self:
+    def _from_config_dicts(self, config_dicts: List[dict], keys: str) -> Self:
         self._from_config_dict(config_dicts[0], keys)
         for config_dict in config_dicts[1:]:
             # if inherited class takes only one argument
@@ -125,79 +125,3 @@ def _from_config_dicts(self, config_dicts, keys: str) -> Self:
             else:
                 self._next = type(self)(config_dict, keys)
             self._next._previous = self  # pylint: disable=protected-access
-
-
-def _example():
-    class NameAsKeyConfig(Config):
-        """Testing config class the uses class name as key"""
-        def __init__(self, config: Union[dict, str]) -> None:
-            """To create a config that uses it's class name when look for nested config,
-            pass None to the super.__init()"""
-            self.idiom = None
-            self.metaphor = None
-            super().__init__(config, None)
-
-    class WithoutKeyConfig(Config):
-        """Testing config class the uses no key"""
-        def __init__(self, config: Union[dict, str]) -> None:
-            """To create a config that does NOT look for config nested under a key,
-            pass an empty string to the super.__init()"""
-            self.idiom = None
-            self.metaphor = None
-            super().__init__(config, '')
-
-    class RequiresKeyConfig(Config):
-        """Testing config class that requires key to be provided"""
-        def __init__(self, config: Union[dict, str], key: Union[list, str]) -> None:
-            """To create a config that requires a user provided name when look for nested config,
-            pass a key as string or list of string to the super.__init()"""
-            self.idiom = None
-            self.metaphor = None
-            super().__init__(config, key)
-
-    def get_config_dict(key: Union[list, str]) -> dict:
-        idioms = ['Out of hand',
-                  'See eye to eye',
-                  'Under the weather',
-                  'Cut the mustard']
-        metaphors = ['blanket of stars',
-                     'weighing on my mind',
-                     'were a glaring light',
-                     'floated down the river']
-        if key is None:
-            return {'idiom': random.choice(idioms),
-                    'metaphor': random.choice(metaphors)}
-        if isinstance(key, str):
-            key = [key]
-        config_dict = {'idiom': random.choice(idioms),
-                       'metaphor': random.choice(metaphors)}
-        for k in reversed(key):
-            config_dict = {k: config_dict}
-        return config_dict
-
-    # example of config that uses class name to identify relevant block of data
-    config_dict = get_config_dict('NameAsKeyConfig')
-    logging.info(config_dict)
-    config = NameAsKeyConfig(config_dict)
-    logging.info('Idiom:', config.idiom)
-    logging.info('Metaphor:', config.metaphor)
-
-    # example of config with block of data at top level
-    config_dict = get_config_dict(None)
-    logging.info(config_dict)
-    config = WithoutKeyConfig(config_dict)
-    logging.info('Idiom:', config.idiom)
-    logging.info('Metaphor:', config.metaphor)
-
-    # example of config with relevant block of data nested
-    config_dict = get_config_dict('NestKey')
-    logging.info(config_dict)
-    config = RequiresKeyConfig(config_dict, 'NestKey')
-    logging.info('Idiom:', config.idiom)
-    logging.info('Metaphor:', config.metaphor)
-
-
-if __name__ == '__main__':
-    import random
-
-    _example()

python/idsse_common/setup.py

@@ -8,6 +8,7 @@
       author='WIDS',
       author_email='@noaa.gov',
       license='MIT',
+      python_requires=">3.11",
       packages=['idsse.common'],
       # packages=['idsse', 'idsse.common'],
       # packages=find_packages(exclude=['contrib', 'docs', 'tests*']),
@@ -16,4 +17,10 @@
         'pint',
         'importlib_metadata',
         ],
+      extras_require={
+        'develop': [
+          'pytest',
+          'pytest-cov',
+        ]
+      },
       zip_safe=False)

python/idsse_common/test/test_config.py

@@ -10,6 +10,8 @@
 # --------------------------------------------------------------------------------
 import json
 import pytest
+from pytest import MonkeyPatch
+from unittest.mock import Mock, mock_open
 
 from idsse.common.config import Config
 
@@ -19,6 +21,7 @@
 def test_load_from_dict_without_key():
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.a_key = None
             super().__init__(config, '')
@@ -30,6 +33,7 @@ def __init__(self, config: dict) -> None:
 def test_load_from_dict_as_string_without_key():
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.some_key = None
             super().__init__(config, '')
@@ -41,6 +45,7 @@ def __init__(self, config: dict) -> None:
 def test_load_from_dict_with_name_key():
     class NameAsKeyConfig(Config):
         """Config class that class name as the key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.best_key = None
             super().__init__(config, None)
@@ -52,17 +57,20 @@ def __init__(self, config: dict) -> None:
 def test_load_from_dict_require_string_key():
     class RequireKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict, key: str) -> None:
             self.some_key = None
             super().__init__(config, key)
 
-    config = RequireKeyConfig('{"custom_key": {"some_key": "value found"}}', 'custom_key')
+    config = RequireKeyConfig(
+        '{"custom_key": {"some_key": "value found"}}', 'custom_key')
     assert config.some_key == 'value found'
 
 
 def test_load_from_dict_require_list_key():
     class RequireKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict, key: list) -> None:
             self.some_key = None
             super().__init__(config, key)
@@ -76,6 +84,7 @@ def __init__(self, config: dict, key: list) -> None:
 def test_load_with_missing_attribute_should_fail():
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.a_key = None
             super().__init__(config, '')
@@ -84,9 +93,40 @@ def __init__(self, config: dict) -> None:
         WithoutKeyConfig({'diff_key': 'value found'})
 
 
+def test_config_str_with_no_files_raises_error(monkeypatch: MonkeyPatch):
+    class WithoutKeyConfig(Config):
+        """Config class that doesn't use a key to find config data"""
+
+        def __init__(self, config: str) -> None:
+            self.a_key = None
+            super().__init__(config, '')
+
+    monkeypatch.setattr('glob.glob', Mock(return_value=[]))
+
+    with pytest.raises(FileNotFoundError):
+        WithoutKeyConfig('wont_be_found')
+
+
+def test_config_list_of_dicts_succeeds():
+    class WithoutKeyConfig(Config):
+        """Config class that doesn't use a key to find config data"""
+
+        def __init__(self, config: dict) -> None:
+            self.a_key = None
+            self.b_key = None
+            super().__init__(config, '', ignore_missing=True)
+
+    config = WithoutKeyConfig(
+        [{'a_key': 'value for a'}, {'b_key': 'value for b'}])
+
+    assert config.a_key == 'value for a'
+    assert config.next.b_key == 'value for b'
+
+
 def test_load_with_ignore_missing_attribute():
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.a_key = None
             self.b_key = None
@@ -96,58 +136,67 @@ def __init__(self, config: dict) -> None:
     assert config.a_key == 'value for a'
 
 
-def test_load_from_file(mocker):
+def test_load_from_file(monkeypatch: MonkeyPatch):
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.y_this_is_a_key = None
             super().__init__(config, '')
 
-    mocker.patch('glob.glob', return_value=['filename'])
+    monkeypatch.setattr('glob.glob', Mock(return_value=['filename']))
 
     read_data = json.dumps({"y_this_is_a_key": "value found in file"})
-    mocker.patch('builtins.open', mocker.mock_open(read_data=read_data))
+    monkeypatch.setattr('builtins.open', mock_open(read_data=read_data))
 
     config = WithoutKeyConfig('path/to/file')
 
     assert config.y_this_is_a_key == "value found in file"
 
 
-def test_load_from_files_with_out_key(mocker):
+def test_load_from_files_with_out_key(monkeypatch: MonkeyPatch):
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.y_this_is_a_key = None
             super().__init__(config, [])
 
-    mocker.patch('glob.glob', return_value=['filename1', 'filename2'])
+    monkeypatch.setattr('glob.glob', Mock(
+        return_value=['filename1', 'filename2']))
 
     read_data = [json.dumps({"y_this_is_a_key": "value found in file1"}),
                  json.dumps({"y_this_is_a_key": "value found in file2"})]
-    mock_files = mocker.patch('builtins.open', mocker.mock_open(read_data=read_data[0]))
-    mock_files.side_effect = (mocker.mock_open(read_data=data).return_value for data in read_data)
+    mock_files = Mock(side_effect=(
+        mock_open(read_data=data).return_value for data in read_data))
+    monkeypatch.setattr('builtins.open', mock_files)
 
     config = WithoutKeyConfig('path/to/dir')
 
     assert config.y_this_is_a_key == "value found in file1"
     assert config.next.y_this_is_a_key == "value found in file2"
+    assert mock_files.call_count == 2
 
 
-def test_load_from_files_with_key(mocker):
+def test_load_from_files_with_key(monkeypatch: MonkeyPatch):
     class WithoutKeyConfig(Config):
         """Config class that doesn't use a key to find config data"""
+
         def __init__(self, config: dict) -> None:
             self.y_this_is_a_key = None
             super().__init__(config, 'config_key')
 
-    mocker.patch('glob.glob', return_value=['filename1', 'filename2'])
+    monkeypatch.setattr('glob.glob', Mock(
+        return_value=['filename1', 'filename2']))
 
     read_data = [json.dumps({"config_key": {"y_this_is_a_key": "value found in file1"}}),
                  json.dumps({"config_key": {"y_this_is_a_key": "value found in file2"}})]
-    mock_files = mocker.patch('builtins.open', mocker.mock_open(read_data=read_data[0]))
-    mock_files.side_effect = (mocker.mock_open(read_data=data).return_value for data in read_data)
+    mock_files = Mock(side_effect=(
+        mock_open(read_data=data).return_value for data in read_data))
+    monkeypatch.setattr('builtins.open', mock_files)
 
     config = WithoutKeyConfig('path/to/dir')
 
     assert config.y_this_is_a_key == "value found in file1"
     assert config.next.y_this_is_a_key == "value found in file2"
+    assert mock_files.call_count == 2

rabbitmq/test/run_test.sh

@@ -4,4 +4,4 @@
 #     [1] = rmq server host name
 #     [2] = rmq client/server username
 #     [3] = rmq client/server password
-python ./test_driver.py "rmqtest" "idss" "password"
+python ./test_rabbitmq_driver.py "rmqtest" "idss" "password"