diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6c6ee58..3f41ed4 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -15,7 +15,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at https://github.com/IBM/firepit/issues. +Report bugs at https://github.com/opencybersecurityalliance/firepit/issues. If you are reporting a bug, please include: @@ -45,7 +45,7 @@ articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://github.com/IBM/firepit/issues. +The best way to send feedback is to file an issue at https://github.com/opencybersecurityalliance/firepit/issues. If you are proposing a feature: @@ -67,7 +67,7 @@ Ready to contribute? Here's how to set up `firepit` for local development. 3. Install your local copy into a virtualenv. Assuming you have pyenv installed, this is how you set up your fork for local development:: $ cd firepit/ - $ pyenv virtualenv python-3.6.8 firepit + $ pyenv virtualenv python-3.9.2 firepit $ pyenv local firepit $ make setup @@ -90,6 +90,8 @@ Ready to contribute? Here's how to set up `firepit` for local development. $ git push origin name-of-your-bugfix-or-feature 7. Submit a pull request through the GitHub website. +8. Request a review from one of the maintainers. + Pull Request Guidelines ----------------------- @@ -100,7 +102,8 @@ Before you submit a pull request, check that it meets these guidelines: 2. If the pull request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring, and add the feature to the list in README.rst. -3. The pull request should work for Python 3.7, 3.8 and 3.9. +3. The pull request should work for Python 3.8, 3.9, 3.10, and 3.11. + There is a GitHub workflow that will check for this automatically. All contributions must be covered by a `Contributor's License Agreement`_ (CLA) and `Entity CLA`_ (if you are contributing on behalf of your employer). You will get a prompt to sign CLA when you submit your first PR. @@ -109,13 +112,17 @@ Tips To run a subset of tests:: -$ pytest tests.test_firepit +$ pytest tests/test_storage.py + +Or:: + +$ pytest -k test_something... -Deploying +Releasing --------- -A reminder for the maintainers on how to deploy. +A reminder for the maintainers on how to release. Make sure all your changes are committed (including an entry in HISTORY.rst). Then run:: diff --git a/firepit/__init__.py b/firepit/__init__.py index a9655d1..9eccab6 100644 --- a/firepit/__init__.py +++ b/firepit/__init__.py @@ -2,7 +2,7 @@ __author__ = """IBM Security""" __email__ = 'pcoccoli@us.ibm.com' -__version__ = '2.3.23' +__version__ = '2.3.24' import re diff --git a/firepit/sqlstorage.py b/firepit/sqlstorage.py index faf159a..911f09c 100644 --- a/firepit/sqlstorage.py +++ b/firepit/sqlstorage.py @@ -910,7 +910,8 @@ def extract_observeddata_attribute( run=True): """ Get the observations of `value` in `viewname`.`path` - Returns list of dicts like {'name_of_SDO_attribute': '...', '{column}': '...'} + Returns list of dicts like {'name_of_attribute': '...', '{column}': '...'} + `name_of_attribute` can be a str or list of str (to get multiple attrs) """ qry = Query([ Table(viewname), @@ -937,11 +938,22 @@ def extract_observeddata_attribute( proj.append(Column(column, table, p)) if column and value is not None: qry.append(Filter([Predicate(column, '=', value)])) - new_col = Column(name_of_attribute, 'observed-data') - qry.append(Order([new_col])) + if isinstance(name_of_attribute, str): + attrs = [name_of_attribute] + elif isinstance(name_of_attribute, list): + attrs = name_of_attribute + else: + raise TypeError('name_of_attribute must be str or list[str]') + new_cols = [] + for attr in attrs: + if attr == 'id': + new_cols.append(Column(attr, 'observed-data', 'observation_id')) + else: + new_cols.append(Column(attr, 'observed-data')) + qry.append(Order(new_cols)) if not proj: proj = [Column('*', viewname)] - qry.append(Projection([new_col] + proj)) + qry.append(Projection(new_cols + proj)) if limit: qry.append(Limit(limit)) diff --git a/setup.cfg b/setup.cfg index a99e922..10620f0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.23 +current_version = 2.3.24 commit = True tag = True diff --git a/setup.py b/setup.py index cd6ae6a..7bf57cd 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ 'anytree', 'python-dateutil', 'ijson', - 'lark-parser', + 'lark', 'tabulate', 'typer', 'ujson' @@ -65,6 +65,6 @@ test_suite='tests', tests_require=test_requirements, url='https://github.com/opencybersecurityalliance/firepit', - version='2.3.23', + version='2.3.24', zip_safe=False, ) diff --git a/tests/test_obs_attr.py b/tests/test_obs_attr.py new file mode 100644 index 0000000..2814c77 --- /dev/null +++ b/tests/test_obs_attr.py @@ -0,0 +1,40 @@ +from .helpers import tmp_storage + + +def test_obs_attr_url(tmpdir, fake_bundle_file): + store = tmp_storage(tmpdir) + store.cache('q1', fake_bundle_file) + data = store.extract_observeddata_attribute( + 'url', + 'last_observed') + assert len(data) == 31 + assert 'last_observed' in data[0] + + +def test_obs_attr_url_only(tmpdir, fake_bundle_file): + store = tmp_storage(tmpdir) + store.cache('q1', fake_bundle_file) + data = store.extract_observeddata_attribute('url', 'id', path=['value']) + assert len(data) == 31 + assert set(data[0].keys()) == {'observation_id', 'value'} + + +def test_obs_attr_multiple(tmpdir, fake_bundle_file): + store = tmp_storage(tmpdir) + store.cache('q1', fake_bundle_file) + attrs = [ + 'number_observed', + 'first_observed', + 'last_observed', + 'id', + ] + data = store.extract_observeddata_attribute('url', attrs) + assert len(data) == 31 + assert set(data[0].keys()) == { + 'number_observed', + 'first_observed', + 'last_observed', + 'observation_id', + 'value', + 'id', # The original url:id + }