Merge pull request #55733 from ogd-software/data_filter_falsey-master

Data filter falsey-master

salt/utils/data.py

@@ -11,6 +11,7 @@
 import fnmatch
 import logging
 import re
+import functools
 
 try:
     from collections.abc import Mapping, MutableMapping, Sequence
@@ -976,3 +977,50 @@ def stringify(data):
             item = six.text_type(item)
         ret.append(item)
     return ret
+
+
+def _is_not_considered_falsey(value, ignore_types=()):
+    '''
+    Helper function for filter_falsey to determine if something is not to be
+    considered falsey.
+    :param any value: The value to consider
+    :param list ignore_types: The types to ignore when considering the value.
+    :return bool
+    '''
+    return isinstance(value, bool) or type(value) in ignore_types or value
+
+
+def filter_falsey(data, recurse_depth=None, ignore_types=()):
+    '''
+    Helper function to remove items from an iterable with falsey value.
+    Removes ``None``, ``{}`` and ``[]``, 0, '' (but does not remove ``False``).
+    Recurses into sub-iterables if ``recurse`` is set to ``True``.
+    :param dict/list data: Source iterable (dict, OrderedDict, list, set, ...) to process.
+    :param int recurse_depth: Recurse this many levels into values that are dicts
+        or lists to also process those. Default: 0 (do not recurse)
+    :param list ignore_types: Contains types that can be falsey but must not
+        be filtered. Default: Only booleans are not filtered.
+    :return type(data)
+    .. version-added:: Neon
+    '''
+    filter_element = (
+        functools.partial(filter_falsey,
+                          recurse_depth=recurse_depth-1,
+                          ignore_types=ignore_types)
+        if recurse_depth else lambda x: x
+    )
+
+    if isinstance(data, dict):
+        processed_elements = [(key, filter_element(value)) for key, value in six.iteritems(data)]
+        return type(data)([
+            (key, value)
+            for key, value in processed_elements
+            if _is_not_considered_falsey(value, ignore_types=ignore_types)
+        ])
+    if hasattr(data, '__iter__') and not isinstance(data, six.string_types):
+        processed_elements = (filter_element(value) for value in data)
+        return type(data)([
+            value for value in processed_elements
+            if _is_not_considered_falsey(value, ignore_types=ignore_types)
+        ])
+    return data

tests/unit/utils/test_data.py

@@ -131,8 +131,8 @@ def test_subdict_match(self):
                 test_three_level_dict, 'a:b:c:v'
             )
         )
-        self.assertFalse(
         # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v'
+        self.assertFalse(
             salt.utils.data.subdict_match(
                 test_three_level_dict, 'a:c:v'
             )
@@ -371,14 +371,17 @@ def test_decode_to_str(self):
             BYTES,
             [123, 456.789, _s('спам'), True, False, None, _s('яйца'), BYTES],
             (987, 654.321, _s('яйца'), _s('яйца'), None, (True, _s('яйца'), BYTES)),
-            {_s('str_key'): _s('str_val'),
-             None: True,
-             123: 456.789,
-             _s('яйца'): BYTES,
-             _s('subdict'): {
-                 _s('unicode_key'): _s('яйца'),
-                 _s('tuple'): (123, _s('hello'), _s('world'), True, _s('яйца'), BYTES),
-                 _s('list'): [456, _s('спам'), False, _s('яйца'), BYTES]}},
+            {
+                _s('str_key'): _s('str_val'),
+                None: True,
+                123: 456.789,
+                _s('яйца'): BYTES,
+                _s('subdict'): {
+                    _s('unicode_key'): _s('яйца'),
+                    _s('tuple'): (123, _s('hello'), _s('world'), True, _s('яйца'), BYTES),
+                    _s('list'): [456, _s('спам'), False, _s('яйца'), BYTES]
+                }
+            },
             OrderedDict([(_s('foo'), _s('bar')), (123, 456), (_s('яйца'), BYTES)])
         ]
 
@@ -466,14 +469,18 @@ def test_encode(self):
             BYTES,
             [123, 456.789, _b('спам'), True, False, None, _b(EGGS), BYTES],
             (987, 654.321, _b('яйца'), _b(EGGS), None, (True, _b(EGGS), BYTES)),
-            {_b('str_key'): _b('str_val'),
-             None: True,
-             123: 456.789,
-             _b(EGGS): BYTES,
-             _b('subdict'): {_b('unicode_key'): _b(EGGS),
-                             _b('tuple'): (123, _b('hello'), _b('world'), True, _b(EGGS), BYTES),
-                             _b('list'): [456, _b('спам'), False, _b(EGGS), BYTES]}},
-             OrderedDict([(_b('foo'), _b('bar')), (123, 456), (_b(EGGS), BYTES)])
+            {
+                _b('str_key'): _b('str_val'),
+                None: True,
+                123: 456.789,
+                _b(EGGS): BYTES,
+                _b('subdict'): {
+                    _b('unicode_key'): _b(EGGS),
+                    _b('tuple'): (123, _b('hello'), _b('world'), True, _b(EGGS), BYTES),
+                    _b('list'): [456, _b('спам'), False, _b(EGGS), BYTES]
+                }
+            },
+            OrderedDict([(_b('foo'), _b('bar')), (123, 456), (_b(EGGS), BYTES)])
         ]
 
         # Both keep=True and keep=False should work because the BYTES data is
@@ -572,7 +579,7 @@ def test_encode_keep(self):
             keep=False,
             preserve_tuples=True)
 
-        for index, item in enumerate(data):
+        for index, _ in enumerate(data):
             self.assertEqual(
                 salt.utils.data.encode(data[index], encoding,
                                        keep=True, preserve_tuples=True),
@@ -620,3 +627,230 @@ def test_stringify(self):
             salt.utils.data.stringify(['one', 'two', str('three'), 4, 5]),  # future lint: disable=blacklisted-function
             ['one', 'two', 'three', '4', '5']
         )
+
+
+class FilterFalseyTestCase(TestCase):
+    '''
+    Test suite for salt.utils.data.filter_falsey
+    '''
+
+    def test_nop(self):
+        '''
+        Test cases where nothing will be done.
+        '''
+        # Test with dictionary without recursion
+        old_dict = {'foo': 'bar', 'bar': {'baz': {'qux': 'quux'}}, 'baz': ['qux', {'foo': 'bar'}]}
+        new_dict = salt.utils.data.filter_falsey(old_dict)
+        self.assertEqual(old_dict, new_dict)
+        # Check returned type equality
+        self.assertIs(type(old_dict), type(new_dict))
+        # Test dictionary with recursion
+        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
+        self.assertEqual(old_dict, new_dict)
+        # Test with list
+        old_list = ['foo', 'bar']
+        new_list = salt.utils.data.filter_falsey(old_list)
+        self.assertEqual(old_list, new_list)
+        # Check returned type equality
+        self.assertIs(type(old_list), type(new_list))
+        # Test with set
+        old_set = set(['foo', 'bar'])
+        new_set = salt.utils.data.filter_falsey(old_set)
+        self.assertEqual(old_set, new_set)
+        # Check returned type equality
+        self.assertIs(type(old_set), type(new_set))
+        # Test with OrderedDict
+        old_dict = OrderedDict([
+            ('foo', 'bar'),
+            ('bar', OrderedDict([('qux', 'quux')])),
+            ('baz', ['qux', OrderedDict([('foo', 'bar')])])
+        ])
+        new_dict = salt.utils.data.filter_falsey(old_dict)
+        self.assertEqual(old_dict, new_dict)
+        self.assertIs(type(old_dict), type(new_dict))
+        # Test excluding int
+        old_list = [0]
+        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type(0)])
+        self.assertEqual(old_list, new_list)
+        # Test excluding str (or unicode) (or both)
+        old_list = ['']
+        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type('')])
+        self.assertEqual(old_list, new_list)
+        # Test excluding list
+        old_list = [[]]
+        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type([])])
+        self.assertEqual(old_list, new_list)
+        # Test excluding dict
+        old_list = [{}]
+        new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})])
+        self.assertEqual(old_list, new_list)
+
+    def test_filter_dict_no_recurse(self):
+        '''
+        Test filtering a dictionary without recursing.
+        This will only filter out key-values where the values are falsey.
+        '''
+        old_dict = {'foo': None,
+                    'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}},
+                    'baz': ['qux'],
+                    'qux': {},
+                    'quux': []}
+        new_dict = salt.utils.data.filter_falsey(old_dict)
+        expect_dict = {'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']}
+        self.assertEqual(expect_dict, new_dict)
+        self.assertIs(type(expect_dict), type(new_dict))
+
+    def test_filter_dict_recurse(self):
+        '''
+        Test filtering a dictionary with recursing.
+        This will filter out any key-values where the values are falsey or when
+        the values *become* falsey after filtering their contents (in case they
+        are lists or dicts).
+        '''
+        old_dict = {'foo': None,
+                    'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}},
+                    'baz': ['qux'],
+                    'qux': {},
+                    'quux': []}
+        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
+        expect_dict = {'baz': ['qux']}
+        self.assertEqual(expect_dict, new_dict)
+        self.assertIs(type(expect_dict), type(new_dict))
+
+    def test_filter_list_no_recurse(self):
+        '''
+        Test filtering a list without recursing.
+        This will only filter out items which are falsey.
+        '''
+        old_list = ['foo', None, [], {}, 0, '']
+        new_list = salt.utils.data.filter_falsey(old_list)
+        expect_list = ['foo']
+        self.assertEqual(expect_list, new_list)
+        self.assertIs(type(expect_list), type(new_list))
+        # Ensure nested values are *not* filtered out.
+        old_list = [
+            'foo',
+            ['foo'],
+            ['foo', None],
+            {'foo': 0},
+            {'foo': 'bar', 'baz': []},
+            [{'foo': ''}],
+        ]
+        new_list = salt.utils.data.filter_falsey(old_list)
+        self.assertEqual(old_list, new_list)
+        self.assertIs(type(old_list), type(new_list))
+
+    def test_filter_list_recurse(self):
+        '''
+        Test filtering a list with recursing.
+        This will filter out any items which are falsey, or which become falsey
+        after filtering their contents (in case they are lists or dicts).
+        '''
+        old_list = [
+            'foo',
+            ['foo'],
+            ['foo', None],
+            {'foo': 0},
+            {'foo': 'bar', 'baz': []},
+            [{'foo': ''}]
+        ]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3)
+        expect_list = ['foo', ['foo'], ['foo'], {'foo': 'bar'}]
+        self.assertEqual(expect_list, new_list)
+        self.assertIs(type(expect_list), type(new_list))
+
+    def test_filter_set_no_recurse(self):
+        '''
+        Test filtering a set without recursing.
+        Note that a set cannot contain unhashable types, so recursion is not possible.
+        '''
+        old_set = set([
+            'foo',
+            None,
+            0,
+            '',
+        ])
+        new_set = salt.utils.data.filter_falsey(old_set)
+        expect_set = set(['foo'])
+        self.assertEqual(expect_set, new_set)
+        self.assertIs(type(expect_set), type(new_set))
+
+    def test_filter_ordereddict_no_recurse(self):
+        '''
+        Test filtering an OrderedDict without recursing.
+        '''
+        old_dict = OrderedDict([
+            ('foo', None),
+            ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])),
+            ('baz', ['qux']),
+            ('qux', {}),
+            ('quux', [])
+        ])
+        new_dict = salt.utils.data.filter_falsey(old_dict)
+        expect_dict = OrderedDict([
+            ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])),
+            ('baz', ['qux']),
+        ])
+        self.assertEqual(expect_dict, new_dict)
+        self.assertIs(type(expect_dict), type(new_dict))
+
+    def test_filter_ordereddict_recurse(self):
+        '''
+        Test filtering an OrderedDict with recursing.
+        '''
+        old_dict = OrderedDict([
+            ('foo', None),
+            ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])),
+            ('baz', ['qux']),
+            ('qux', {}),
+            ('quux', [])
+        ])
+        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3)
+        expect_dict = OrderedDict([
+            ('baz', ['qux']),
+        ])
+        self.assertEqual(expect_dict, new_dict)
+        self.assertIs(type(expect_dict), type(new_dict))
+
+    def test_filter_list_recurse_limit(self):
+        '''
+        Test filtering a list with recursing, but with a limited depth.
+        Note that the top-level is always processed, so a recursion depth of 2
+        means that two *additional* levels are processed.
+        '''
+        old_list = [None, [None, [None, [None]]]]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2)
+        self.assertEqual([[[[None]]]], new_list)
+
+    def test_filter_dict_recurse_limit(self):
+        '''
+        Test filtering a dict with recursing, but with a limited depth.
+        Note that the top-level is always processed, so a recursion depth of 2
+        means that two *additional* levels are processed.
+        '''
+        old_dict = {'one': None,
+                    'foo': {'two': None, 'bar': {'three': None, 'baz': {'four': None}}}}
+        new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2)
+        self.assertEqual({'foo': {'bar': {'baz': {'four': None}}}}, new_dict)
+
+    def test_filter_exclude_types(self):
+        '''
+        Test filtering a list recursively, but also ignoring (i.e. not filtering)
+        out certain types that can be falsey.
+        '''
+        # Ignore int, unicode
+        old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(0), type('')])
+        self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 0}, {'foo': 'bar'}, [{'foo': ''}]], new_list)
+        # Ignore list
+        old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type([])])
+        self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar', 'baz': []}, []], new_list)
+        # Ignore dict
+        old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type({})])
+        self.assertEqual(['foo', ['foo'], ['foo'], {}, {'foo': 'bar'}, [{}]], new_list)
+        # Ignore NoneType
+        old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]]
+        new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(None)])
+        self.assertEqual(['foo', ['foo'], ['foo', None], {'foo': 'bar'}], new_list)