From 69391ffb17f1094bae80961d79437b4e9a527670 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Thu, 11 Apr 2019 15:40:12 +0200 Subject: [PATCH 1/6] Add filter_falsey function to filter falsey values from a list or dict. --- salt/utils/data.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/salt/utils/data.py b/salt/utils/data.py index 2e818dbcdcf3..2ccd60ad5018 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -993,3 +993,40 @@ def json_query(data, expr): log.error(err) raise RuntimeError(err) return jmespath.search(expr, data) + + +def filter_falsey(data, recurse_depth=None, ignore_types=None): + ''' + Helper function to remove items from a dict or list with falsey value. + Removes ``None``, ``{}`` and ``[]``, 0, '' (but does not remove ``False``). + Recurses into subdicts or sublists if ``recurse`` is set to ``True``. + + :param dict/list data: Source dict or list to process. + :param int recurse_depth: Recurse this many levels into values that are dicts + or lists to also process those. Default: 0 (do not recurse) + :param list ignore_types: Contains types that can be falsey but must not + be filtered. Default: Only booleans are not filtered. + + :return type(data) + ''' + if ignore_types is None: + ignore_types = [] + if isinstance(data, dict): + ret = {} + for key, value in six.iteritems(data): + new_value = filter_falsey(value, + recurse_depth=recurse_depth-1, + ignore_types=ignore_types) if recurse_depth else value + if isinstance(new_value, bool) or (type(new_value) in ignore_types) or new_value: + ret.update({key: new_value}) + elif isinstance(data, list): + ret = [] + for item in data: + new_value = filter_falsey(item, + recurse_depth=recurse_depth-1, + ignore_types=ignore_types) if recurse_depth else item + if isinstance(new_value, bool) or (type(new_value) in ignore_types) or new_value: + ret.append(new_value) + else: + ret = data + return ret From 2335ca10df9feb83d1fa1a7ccf0c397558a1ce27 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Thu, 11 Apr 2019 15:40:36 +0200 Subject: [PATCH 2/6] Add unit tests for the filter_falsey function. --- tests/unit/utils/test_data.py | 120 ++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py index 1d6a248e8769..e6929c7ad1eb 100644 --- a/tests/unit/utils/test_data.py +++ b/tests/unit/utils/test_data.py @@ -619,3 +619,123 @@ def test_json_query(self): sorted(salt.utils.data.json_query(user_groups, expression)), primary_groups ) + + +class FilterEmptyValuesTestCase(TestCase): + + def test_nop(self): + ''' + Test cases where nothing will be done. + ''' + # Test with dictionary + old_dict = {'foo': 'bar', 'bar': {'baz': {'qux': 'quux'}}, 'baz': ['qux', {'foo': 'bar'}]} + new_dict = salt.utils.data.filter_falsey(old_dict) + self.assertEqual(old_dict, new_dict) + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + self.assertEqual(old_dict, new_dict) + # Test with list + old_list = ['foo', 'bar'] + new_list = salt.utils.data.filter_falsey(old_list) + self.assertEqual(old_list, new_list) + # Test excluding int + old_list = [0] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type(0)]) + self.assertEqual(old_list, new_list) + # Test excluding str (or unicode) (or both) + old_list = [''] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type('')]) + self.assertEqual(old_list, new_list) + # Test excluding list + old_list = [[]] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type([])]) + self.assertEqual(old_list, new_list) + # Test excluding dict + old_list = [{}] + new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})]) + self.assertEqual(old_list, new_list) + + + def test_filter_dict_no_recurse(self): + ''' + Test filtering a dictionary without recursing. + This will only filter out key-values where the values are falsey. + ''' + old_dict = {'foo': None, 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux'], 'qux': {}, 'quux': []} + new_dict = salt.utils.data.filter_falsey(old_dict) + self.assertEqual({'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']}, new_dict) + + def test_filter_dict_recurse(self): + ''' + Test filtering a dictionary with recursing. + This will filter out any key-values where the values are falsey or when + the values *become* falsey after filtering their contents (in case they + are lists or dicts). + ''' + old_dict = {'foo': None, 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux'], 'qux': {}, 'quux': []} + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + self.assertEqual({'baz': ['qux']}, new_dict) + + def test_filter_list_no_recurse(self): + ''' + Test filtering a list without recursing. + This will only filter out items which are falsey. + ''' + old_list = ['foo', None, [], {}, 0, ''] + new_list = salt.utils.data.filter_falsey(old_list) + self.assertEqual(['foo'], new_list) + # Ensure nested values are *not* filtered out. + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list) + self.assertEqual(old_list, new_list) + + def test_filter_list_recurse(self): + ''' + Test filtering a list with recursing. + This will filter out any items which are falsey, or which become falsey + after filtering their contents (in case they are lists or dicts). + ''' + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) + self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar'}], new_list) + + def test_filter_list_recurse_limit(self): + ''' + Test filtering a list with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. + ''' + old_list = [None, [None, [None, [None]]]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=2) + self.assertEqual([[[[None]]]], new_list) + + def test_filter_dict_recurse_limit(self): + ''' + Test filtering a dict with recursing, but with a limited depth. + Note that the top-level is always processed, so a recursion depth of 2 + means that two *additional* levels are processed. + ''' + old_dict = {'one': None, 'foo': {'two': None, 'bar': {'three': None, 'baz': {'four': None}}}} + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2) + self.assertEqual({'foo': {'bar': {'baz': {'four': None}}}}, new_dict) + + def test_filter_exclude_types(self): + ''' + Test filtering a list recursively, but also ignoring (i.e. not filtering) + out certain types that can be falsey. + ''' + # Ignore int, unicode + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(0), type('')]) + self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 0}, {'foo': 'bar'}, [{'foo': ''}]], new_list) + # Ignore list + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type([])]) + self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar', 'baz': []}, []], new_list) + # Ignore dict + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type({})]) + self.assertEqual(['foo', ['foo'], ['foo'], {}, {'foo': 'bar'}, [{}]], new_list) + # Ignore NoneType + old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3, ignore_types=[type(None)]) + self.assertEqual(['foo', ['foo'], ['foo', None], {'foo': 'bar'}], new_list) From 0a6e505e604b0313fb6ecc54b1de2db9cdfaaac4 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Thu, 11 Apr 2019 16:09:44 +0200 Subject: [PATCH 3/6] Forgot to pylint the unit-tests. Pylint-inspired fixes. --- tests/unit/utils/test_data.py | 68 +++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py index e6929c7ad1eb..a0ed87603d26 100644 --- a/tests/unit/utils/test_data.py +++ b/tests/unit/utils/test_data.py @@ -131,8 +131,8 @@ def test_subdict_match(self): test_three_level_dict, 'a:b:c:v' ) ) - self.assertFalse( # Test regression in 2015.8 where 'a:c:v' would match 'a:b:c:v' + self.assertFalse( salt.utils.data.subdict_match( test_three_level_dict, 'a:c:v' ) @@ -443,14 +443,18 @@ def test_encode(self): BYTES, [123, 456.789, _b('спам'), True, False, None, _b(EGGS), BYTES], (987, 654.321, _b('яйца'), _b(EGGS), None, (True, _b(EGGS), BYTES)), - {_b('str_key'): _b('str_val'), - None: True, - 123: 456.789, - _b(EGGS): BYTES, - _b('subdict'): {_b('unicode_key'): _b(EGGS), - _b('tuple'): (123, _b('hello'), _b('world'), True, _b(EGGS), BYTES), - _b('list'): [456, _b('спам'), False, _b(EGGS), BYTES]}}, - OrderedDict([(_b('foo'), _b('bar')), (123, 456), (_b(EGGS), BYTES)]) + { + _b('str_key'): _b('str_val'), + None: True, + 123: 456.789, + _b(EGGS): BYTES, + _b('subdict'): { + _b('unicode_key'): _b(EGGS), + _b('tuple'): (123, _b('hello'), _b('world'), True, _b(EGGS), BYTES), + _b('list'): [456, _b('спам'), False, _b(EGGS), BYTES] + } + }, + OrderedDict([(_b('foo'), _b('bar')), (123, 456), (_b(EGGS), BYTES)]) ] # Both keep=True and keep=False should work because the BYTES data is @@ -549,7 +553,7 @@ def test_encode_keep(self): keep=False, preserve_tuples=True) - for index, item in enumerate(data): + for index, _ in enumerate(data): self.assertEqual( salt.utils.data.encode(data[index], encoding, keep=True, preserve_tuples=True), @@ -621,7 +625,10 @@ def test_json_query(self): ) -class FilterEmptyValuesTestCase(TestCase): +class FilterFalseyTestCase(TestCase): + ''' + Test suite for salt.utils.data.filter_falsey + ''' def test_nop(self): ''' @@ -654,15 +661,21 @@ def test_nop(self): new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type({})]) self.assertEqual(old_list, new_list) - def test_filter_dict_no_recurse(self): ''' Test filtering a dictionary without recursing. This will only filter out key-values where the values are falsey. ''' - old_dict = {'foo': None, 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux'], 'qux': {}, 'quux': []} + old_dict = {'foo': None, + 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, + 'baz': ['qux'], + 'qux': {}, + 'quux': []} new_dict = salt.utils.data.filter_falsey(old_dict) - self.assertEqual({'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']}, new_dict) + self.assertEqual( + {'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']}, + new_dict + ) def test_filter_dict_recurse(self): ''' @@ -671,7 +684,11 @@ def test_filter_dict_recurse(self): the values *become* falsey after filtering their contents (in case they are lists or dicts). ''' - old_dict = {'foo': None, 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux'], 'qux': {}, 'quux': []} + old_dict = {'foo': None, + 'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, + 'baz': ['qux'], + 'qux': {}, + 'quux': []} new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) self.assertEqual({'baz': ['qux']}, new_dict) @@ -684,7 +701,14 @@ def test_filter_list_no_recurse(self): new_list = salt.utils.data.filter_falsey(old_list) self.assertEqual(['foo'], new_list) # Ensure nested values are *not* filtered out. - old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + old_list = [ + 'foo', + ['foo'], + ['foo', None], + {'foo': 0}, + {'foo': 'bar', 'baz': []}, + [{'foo': ''}], + ] new_list = salt.utils.data.filter_falsey(old_list) self.assertEqual(old_list, new_list) @@ -694,7 +718,14 @@ def test_filter_list_recurse(self): This will filter out any items which are falsey, or which become falsey after filtering their contents (in case they are lists or dicts). ''' - old_list = ['foo', ['foo'], ['foo', None], {'foo': 0}, {'foo': 'bar', 'baz': []}, [{'foo': ''}]] + old_list = [ + 'foo', + ['foo'], + ['foo', None], + {'foo': 0}, + {'foo': 'bar', 'baz': []}, + [{'foo': ''}] + ] new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar'}], new_list) @@ -714,7 +745,8 @@ def test_filter_dict_recurse_limit(self): Note that the top-level is always processed, so a recursion depth of 2 means that two *additional* levels are processed. ''' - old_dict = {'one': None, 'foo': {'two': None, 'bar': {'three': None, 'baz': {'four': None}}}} + old_dict = {'one': None, + 'foo': {'two': None, 'bar': {'three': None, 'baz': {'four': None}}}} new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=2) self.assertEqual({'foo': {'bar': {'baz': {'four': None}}}}, new_dict) From 89b66a73077504365924eb7a5b5be8fe38a5c681 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Fri, 12 Apr 2019 09:38:13 +0200 Subject: [PATCH 4/6] Added version-added docstring to filter_falsey function. --- salt/utils/data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/salt/utils/data.py b/salt/utils/data.py index 2ccd60ad5018..012c8065731c 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -1008,6 +1008,8 @@ def filter_falsey(data, recurse_depth=None, ignore_types=None): be filtered. Default: Only booleans are not filtered. :return type(data) + + .. version-added:: Neon ''' if ignore_types is None: ignore_types = [] From 815c6ccae75d8375a602175d380a450b54983743 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Fri, 12 Apr 2019 13:36:26 +0200 Subject: [PATCH 5/6] Rewritten filter_falsey to allow for more iterables than just dict and list. Updated tests with type-checks. --- salt/utils/data.py | 62 ++++++++++++++--------- tests/unit/utils/test_data.py | 95 ++++++++++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 32 deletions(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index 012c8065731c..1613971de8d2 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -11,6 +11,7 @@ import fnmatch import logging import re +import functools try: from collections.abc import Mapping, MutableMapping, Sequence @@ -995,13 +996,26 @@ def json_query(data, expr): return jmespath.search(expr, data) -def filter_falsey(data, recurse_depth=None, ignore_types=None): +def _is_not_considered_falsey(value, ignore_types=()): ''' - Helper function to remove items from a dict or list with falsey value. + Helper function for filter_falsey to determine if something is not to be + considered falsey. + + :param any value: The value to consider + :param list ignore_types: The types to ignore when considering the value. + + :return bool + ''' + return isinstance(value, bool) or type(value) in ignore_types or value + + +def filter_falsey(data, recurse_depth=None, ignore_types=()): + ''' + Helper function to remove items from an iterable with falsey value. Removes ``None``, ``{}`` and ``[]``, 0, '' (but does not remove ``False``). - Recurses into subdicts or sublists if ``recurse`` is set to ``True``. + Recurses into sub-iterables if ``recurse`` is set to ``True``. - :param dict/list data: Source dict or list to process. + :param dict/list data: Source iterable (dict, OrderedDict, list, set, ...) to process. :param int recurse_depth: Recurse this many levels into values that are dicts or lists to also process those. Default: 0 (do not recurse) :param list ignore_types: Contains types that can be falsey but must not @@ -1011,24 +1025,24 @@ def filter_falsey(data, recurse_depth=None, ignore_types=None): .. version-added:: Neon ''' - if ignore_types is None: - ignore_types = [] + filter_element = ( + functools.partial(filter_falsey, + recurse_depth=recurse_depth-1, + ignore_types=ignore_types) + if recurse_depth else lambda x: x + ) + if isinstance(data, dict): - ret = {} - for key, value in six.iteritems(data): - new_value = filter_falsey(value, - recurse_depth=recurse_depth-1, - ignore_types=ignore_types) if recurse_depth else value - if isinstance(new_value, bool) or (type(new_value) in ignore_types) or new_value: - ret.update({key: new_value}) - elif isinstance(data, list): - ret = [] - for item in data: - new_value = filter_falsey(item, - recurse_depth=recurse_depth-1, - ignore_types=ignore_types) if recurse_depth else item - if isinstance(new_value, bool) or (type(new_value) in ignore_types) or new_value: - ret.append(new_value) - else: - ret = data - return ret + processed_elements = [(key, filter_element(value)) for key, value in six.iteritems(data)] + return type(data)([ + (key, value) + for key, value in processed_elements + if _is_not_considered_falsey(value, ignore_types=ignore_types) + ]) + elif hasattr(data, '__iter__'): + processed_elements = (filter_element(value) for value in data) + return type(data)([ + value for value in processed_elements + if _is_not_considered_falsey(value, ignore_types=ignore_types) + ]) + return data diff --git a/tests/unit/utils/test_data.py b/tests/unit/utils/test_data.py index a0ed87603d26..fc15cf27fde0 100644 --- a/tests/unit/utils/test_data.py +++ b/tests/unit/utils/test_data.py @@ -634,16 +634,36 @@ def test_nop(self): ''' Test cases where nothing will be done. ''' - # Test with dictionary + # Test with dictionary without recursion old_dict = {'foo': 'bar', 'bar': {'baz': {'qux': 'quux'}}, 'baz': ['qux', {'foo': 'bar'}]} new_dict = salt.utils.data.filter_falsey(old_dict) self.assertEqual(old_dict, new_dict) + # Check returned type equality + self.assertIs(type(old_dict), type(new_dict)) + # Test dictionary with recursion new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) self.assertEqual(old_dict, new_dict) # Test with list old_list = ['foo', 'bar'] new_list = salt.utils.data.filter_falsey(old_list) self.assertEqual(old_list, new_list) + # Check returned type equality + self.assertIs(type(old_list), type(new_list)) + # Test with set + old_set = set(['foo', 'bar']) + new_set = salt.utils.data.filter_falsey(old_set) + self.assertEqual(old_set, new_set) + # Check returned type equality + self.assertIs(type(old_set), type(new_set)) + # Test with OrderedDict + old_dict = OrderedDict([ + ('foo', 'bar'), + ('bar', OrderedDict([('qux', 'quux')])), + ('baz', ['qux', OrderedDict([('foo', 'bar')])]) + ]) + new_dict = salt.utils.data.filter_falsey(old_dict) + self.assertEqual(old_dict, new_dict) + self.assertIs(type(old_dict), type(new_dict)) # Test excluding int old_list = [0] new_list = salt.utils.data.filter_falsey(old_list, ignore_types=[type(0)]) @@ -672,10 +692,9 @@ def test_filter_dict_no_recurse(self): 'qux': {}, 'quux': []} new_dict = salt.utils.data.filter_falsey(old_dict) - self.assertEqual( - {'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']}, - new_dict - ) + expect_dict = {'bar': {'baz': {'qux': None, 'quux': '', 'foo': []}}, 'baz': ['qux']} + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) def test_filter_dict_recurse(self): ''' @@ -690,7 +709,9 @@ def test_filter_dict_recurse(self): 'qux': {}, 'quux': []} new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) - self.assertEqual({'baz': ['qux']}, new_dict) + expect_dict = {'baz': ['qux']} + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) def test_filter_list_no_recurse(self): ''' @@ -699,7 +720,9 @@ def test_filter_list_no_recurse(self): ''' old_list = ['foo', None, [], {}, 0, ''] new_list = salt.utils.data.filter_falsey(old_list) - self.assertEqual(['foo'], new_list) + expect_list = ['foo'] + self.assertEqual(expect_list, new_list) + self.assertIs(type(expect_list), type(new_list)) # Ensure nested values are *not* filtered out. old_list = [ 'foo', @@ -711,6 +734,7 @@ def test_filter_list_no_recurse(self): ] new_list = salt.utils.data.filter_falsey(old_list) self.assertEqual(old_list, new_list) + self.assertIs(type(old_list), type(new_list)) def test_filter_list_recurse(self): ''' @@ -727,7 +751,62 @@ def test_filter_list_recurse(self): [{'foo': ''}] ] new_list = salt.utils.data.filter_falsey(old_list, recurse_depth=3) - self.assertEqual(['foo', ['foo'], ['foo'], {'foo': 'bar'}], new_list) + expect_list = ['foo', ['foo'], ['foo'], {'foo': 'bar'}] + self.assertEqual(expect_list, new_list) + self.assertIs(type(expect_list), type(new_list)) + + def test_filter_set_no_recurse(self): + ''' + Test filtering a set without recursing. + Note that a set cannot contain unhashable types, so recursion is not possible. + ''' + old_set = set([ + 'foo', + None, + 0, + '', + ]) + new_set = salt.utils.data.filter_falsey(old_set) + expect_set = set(['foo']) + self.assertEqual(expect_set, new_set) + self.assertIs(type(expect_set), type(new_set)) + + def test_filter_ordereddict_no_recurse(self): + ''' + Test filtering an OrderedDict without recursing. + ''' + old_dict = OrderedDict([ + ('foo', None), + ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), + ('baz', ['qux']), + ('qux', {}), + ('quux', []) + ]) + new_dict = salt.utils.data.filter_falsey(old_dict) + expect_dict = OrderedDict([ + ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), + ('baz', ['qux']), + ]) + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) + + def test_filter_ordereddict_recurse(self): + ''' + Test filtering an OrderedDict with recursing. + ''' + old_dict = OrderedDict([ + ('foo', None), + ('bar', OrderedDict([('baz', OrderedDict([('qux', None), ('quux', ''), ('foo', [])]))])), + ('baz', ['qux']), + ('qux', {}), + ('quux', []) + ]) + new_dict = salt.utils.data.filter_falsey(old_dict, recurse_depth=3) + expect_dict = OrderedDict([ + ('baz', ['qux']), + ]) + self.assertEqual(expect_dict, new_dict) + self.assertIs(type(expect_dict), type(new_dict)) def test_filter_list_recurse_limit(self): ''' From 2c3944be6796c741ec8e771cac36123ce8fd15b1 Mon Sep 17 00:00:00 2001 From: Herbert Buurman Date: Fri, 12 Apr 2019 14:38:39 +0200 Subject: [PATCH 6/6] Excluded string types from being handled as an iterator. This fixes unittests under python3. --- salt/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salt/utils/data.py b/salt/utils/data.py index 1613971de8d2..516dac618b27 100644 --- a/salt/utils/data.py +++ b/salt/utils/data.py @@ -1039,7 +1039,7 @@ def filter_falsey(data, recurse_depth=None, ignore_types=()): for key, value in processed_elements if _is_not_considered_falsey(value, ignore_types=ignore_types) ]) - elif hasattr(data, '__iter__'): + elif hasattr(data, '__iter__') and not isinstance(data, six.string_types): processed_elements = (filter_element(value) for value in data) return type(data)([ value for value in processed_elements