Skip to content

Commit

Permalink
fixes #426, adding docs for
Browse files Browse the repository at this point in the history
group_by
  • Loading branch information
seperman committed Oct 17, 2023
1 parent ade098a commit 84fcc41
Showing 9 changed files with 216 additions and 38 deletions.
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -23,9 +23,15 @@ Tested on Python 3.7+ and PyPy3.

Please check the [ChangeLog](CHANGELOG.md) file for the detailed information.

DeepDiff 6-6-1
- Fix for [DeepDiff raises decimal exception when using significant digits](https://github.com/seperman/deepdiff/issues/426)
- Introducing group_by_sort_key
- Adding group_by 2D. For example `group_by=['last_name', 'zip_code']`


DeepDiff 6-6-0

- [Serialize To Flat Dicts]()
- [Serialize To Flat Dicts](https://zepworks.com/deepdiff/current/serialization.html#delta-to-flat-dicts-label)
- [NumPy 2.0 compatibility](https://github.com/seperman/deepdiff/pull/422) by [William Jamieson](https://github.com/WilliamJamieson)

DeepDiff 6-5-0
57 changes: 43 additions & 14 deletions deepdiff/diff.py
Original file line number Diff line number Diff line change
@@ -1601,35 +1601,64 @@ def _get_view_results(self, view):
raise ValueError(INVALID_VIEW_MSG.format(view))
return result

@staticmethod
def _get_key_for_group_by(row, group_by, item_name):
try:
return row.pop(group_by)
except KeyError:
logger.error("Unable to group {} by {}. The key is missing in {}".format(item_name, group_by, row))
raise

def _group_iterable_to_dict(self, item, group_by, item_name):
"""
Convert a list of dictionaries into a dictionary of dictionaries
where the key is the value of the group_by key in each dictionary.
"""
group_by_level2 = None
if isinstance(group_by, (list, tuple)):
group_by_level1 = group_by[0]
if len(group_by) > 1:
group_by_level2 = group_by[1]
else:
group_by_level1 = group_by
if isinstance(item, Iterable) and not isinstance(item, Mapping):
result = {}
item_copy = deepcopy(item)
for row in item_copy:
if isinstance(row, Mapping):
try:
key = row.pop(group_by)
except KeyError:
logger.error("Unable to group {} by {}. The key is missing in {}".format(item_name, group_by, row))
raise
if self.group_by_sort_key:
if key not in result:
result[key] = []
if row not in result[key]:
result[key].append(row)
key1 = self._get_key_for_group_by(row, group_by_level1, item_name)
if group_by_level2:
key2 = self._get_key_for_group_by(row, group_by_level2, item_name)
if key1 not in result:
result[key1] = {}
if self.group_by_sort_key:
if key2 not in result[key1]:
result[key1][key2] = []
result_key1_key2 = result[key1][key2]
if row not in result_key1_key2:
result_key1_key2.append(row)
else:
result[key1][key2] = row
else:
result[key] = row
if self.group_by_sort_key:
if key1 not in result:
result[key1] = []
if row not in result[key1]:
result[key1].append(row)
else:
result[key1] = row
else:
msg = "Unable to group {} by {} since the item {} is not a dictionary.".format(item_name, group_by, row)
msg = "Unable to group {} by {} since the item {} is not a dictionary.".format(item_name, group_by_level1, row)
logger.error(msg)
raise ValueError(msg)
if self.group_by_sort_key:
for key, row in result.items():
row.sort(key=self.group_by_sort_key)
if group_by_level2:
for key1, row1 in result.items():
for key2, row in row1.items():
row.sort(key=self.group_by_sort_key)
else:
for key, row in result.items():
row.sort(key=self.group_by_sort_key)
return result
msg = "Unable to group {} by {}".format(item_name, group_by)
logger.error(msg)
10 changes: 8 additions & 2 deletions deepdiff/helper.py
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@
import string
import time
from ast import literal_eval
from decimal import Decimal, localcontext
from decimal import Decimal, localcontext, InvalidOperation as InvalidDecimalOperation
from collections import namedtuple
from itertools import repeat
from ordered_set import OrderedSet
@@ -394,7 +394,13 @@ def number_to_string(number, significant_digits, number_format_notation="f"):
# Precision = number of integer digits + significant_digits
# Using number//1 to get the integer part of the number
ctx.prec = len(str(abs(number // 1))) + significant_digits
number = number.quantize(Decimal('0.' + '0' * significant_digits))
try:
number = number.quantize(Decimal('0.' + '0' * significant_digits))
except InvalidDecimalOperation:
# Sometimes rounding up causes a higher precision to be needed for the quantize operation
# For example '999.99999999' will become '1000.000000' after quantize
ctx.prec += 1
number = number.quantize(Decimal('0.' + '0' * significant_digits))
elif isinstance(number, only_complex_number):
# Case for complex numbers.
number = number.__class__(
1 change: 1 addition & 0 deletions deepdiff/serialization.py
Original file line number Diff line number Diff line change
@@ -537,6 +537,7 @@ def _serialize_decimal(value):
JSON_CONVERTOR = {
decimal.Decimal: _serialize_decimal,
ordered_set.OrderedSet: list,
set: list,
type: lambda x: x.__name__,
bytes: lambda x: x.decode('utf-8'),
datetime.datetime: lambda x: x.isoformat(),
89 changes: 87 additions & 2 deletions docs/basics.rst
Original file line number Diff line number Diff line change
@@ -148,9 +148,24 @@ Object attribute added:
Group By
--------

group_by can be used when dealing with list of dictionaries to convert them to group them by value defined in group_by. The common use case is when reading data from a flat CSV and primary key is one of the columns in the CSV. We want to use the primary key to group the rows instead of CSV row number.
group_by can be used when dealing with the list of dictionaries. It converts them from lists to a single dictionary with the key defined by group_by. The common use case is when reading data from a flat CSV, and the primary key is one of the columns in the CSV. We want to use the primary key instead of the CSV row number to group the rows. The group_by can do 2D group_by by passing a list of 2 keys.

Example:
For example:
>>> [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'},
... {'id': 'BB', 'name': 'James', 'last_name': 'Blue'},
... {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'},
... ]

Becomes:
>>> t1 = {
... 'AA': {'name': 'Joe', 'last_name': 'Nobody'},
... 'BB': {'name': 'James', 'last_name': 'Blue'},
... 'CC': {'name': 'Mike', 'last_name': 'Apple'},
... }


With that in mind, let's take a look at the following:
>>> from deepdiff import DeepDiff
>>> t1 = [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'},
@@ -187,5 +202,75 @@ Now we use group_by='id':
>>> diff['values_changed'][0].up.up.t1
{'AA': {'name': 'Joe', 'last_name': 'Nobody'}, 'BB': {'name': 'James', 'last_name': 'Blue'}, 'CC': {'name': 'Mike', 'last_name': 'Apple'}}

2D Example:
>>> from pprint import pprint
>>> from deepdiff import DeepDiff
>>>
>>> t1 = [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'},
... {'id': 'BB', 'name': 'James', 'last_name': 'Blue'},
... {'id': 'BB', 'name': 'Jimmy', 'last_name': 'Red'},
... {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'},
... ]
>>>
>>> t2 = [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody'},
... {'id': 'BB', 'name': 'James', 'last_name': 'Brown'},
... {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple'},
... ]
>>>
>>> diff = DeepDiff(t1, t2, group_by=['id', 'name'])
>>> pprint(diff)
{'dictionary_item_removed': [root['BB']['Jimmy']],
'values_changed': {"root['BB']['James']['last_name']": {'new_value': 'Brown',
'old_value': 'Blue'}}}

.. _group_by_sort_key_label:

Group By - Sort Key
-------------------

group_by_sort_key is used to define how dictionaries are sorted if multiple ones fall under one group. When this parameter is used, group_by converts the lists of dictionaries into a dictionary of keys to lists of dictionaries. Then, group_by_sort_key is used to sort between the list.

For example, there are duplicate id values. If we only use group_by='id', one of the dictionaries with id of 'BB' will overwrite the other. However, if we also set group_by_sort_key='name', we keep both dictionaries with the id of 'BB'.

Example:

[{'id': 'AA', 'int_id': 2, 'last_name': 'Nobody', 'name': 'Joe'},
{'id': 'BB', 'int_id': 20, 'last_name': 'Blue', 'name': 'James'},
{'id': 'BB', 'int_id': 3, 'last_name': 'Red', 'name': 'Jimmy'},
{'id': 'CC', 'int_id': 4, 'last_name': 'Apple', 'name': 'Mike'}]


Becomes:
{'AA': [{'int_id': 2, 'last_name': 'Nobody', 'name': 'Joe'}],
'BB': [{'int_id': 20, 'last_name': 'Blue', 'name': 'James'},
{'int_id': 3, 'last_name': 'Red', 'name': 'Jimmy'}],
'CC': [{'int_id': 4, 'last_name': 'Apple', 'name': 'Mike'}]}


Example of using group_by_sort_key
>>> t1 = [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2},
... {'id': 'BB', 'name': 'James', 'last_name': 'Blue', 'int_id': 20},
... {'id': 'BB', 'name': 'Jimmy', 'last_name': 'Red', 'int_id': 3},
... {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4},
... ]
>>>
>>> t2 = [
... {'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2},
... {'id': 'BB', 'name': 'James', 'last_name': 'Brown', 'int_id': 20},
... {'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4},
... ]
>>>
>>> diff = DeepDiff(t1, t2, group_by='id', group_by_sort_key='name')
>>>
>>> pprint(diff)
{'iterable_item_removed': {"root['BB'][1]": {'int_id': 3,
'last_name': 'Red',
'name': 'Jimmy'}},
'values_changed': {"root['BB'][0]['last_name']": {'new_value': 'Brown',
'old_value': 'Blue'}}}


Back to :doc:`/index`
7 changes: 5 additions & 2 deletions docs/diff_doc.rst
Original file line number Diff line number Diff line change
@@ -79,8 +79,11 @@ include_obj_callback_strict: function, default = None
get_deep_distance: Boolean, default = False
:ref:`get_deep_distance_label` will get you the deep distance between objects. The distance is a number between 0 and 1 where zero means there is no diff between the 2 objects and 1 means they are very different. Note that this number should only be used to compare the similarity of 2 objects and nothing more. The algorithm for calculating this number may or may not change in the future releases of DeepDiff.

group_by: String, default=None
:ref:`group_by_label` can be used when dealing with list of dictionaries to convert them to group them by value defined in group_by. The common use case is when reading data from a flat CSV and primary key is one of the columns in the CSV. We want to use the primary key to group the rows instead of CSV row number.
group_by: String or a list of size 2, default=None
:ref:`group_by_label` can be used when dealing with the list of dictionaries. It converts them from lists to a single dictionary with the key defined by group_by. The common use case is when reading data from a flat CSV, and the primary key is one of the columns in the CSV. We want to use the primary key instead of the CSV row number to group the rows. The group_by can do 2D group_by by passing a list of 2 keys.

group_by_sort_key: String or a function
:ref:`group_by_sort_key_label` is used to define how dictionaries are sorted if multiple ones fall under one group. When this parameter is used, group_by converts the lists of dictionaries into a dictionary of keys to lists of dictionaries. Then, :ref:`group_by_sort_key_label` is used to sort between the list.

hasher: default = DeepHash.sha256hex
Hash function to be used. If you don't want SHA256, you can use your own hash function
9 changes: 9 additions & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
@@ -31,6 +31,15 @@ The DeepDiff library includes the following modules:
What Is New
***********

DeepDiff 6-6-1
--------------

- Fix for `DeepDiff raises decimal exception when using significant
digits <https://github.com/seperman/deepdiff/issues/426>`__
- Introducing group_by_sort_key
- Adding group_by 2D. For example
``group_by=['last_name', 'zip_code']``

DeepDiff 6-6-0
--------------

58 changes: 47 additions & 11 deletions tests/test_diff_text.py
Original file line number Diff line number Diff line change
@@ -1144,18 +1144,19 @@ def test_int_to_unicode(self):
}
assert result == ddiff

@pytest.mark.parametrize("t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result", [
(43.265798602382986, 43.71677762295505, False, 0, "f", {'values_changed': {'root': {'new_value': 43.71677762295505, 'old_value': 43.265798602382986}}}), # Note that it rounds the number so one becomes 43 and the other one is 44
(Decimal('2.5'), Decimal('1.5'), False, 0, "f", {}),
(Decimal('2.5'), Decimal('1.5'), False, 1, "f", {'values_changed': {'root': {'new_value': Decimal('1.5'), 'old_value': Decimal('2.5')}}}),
(Decimal('2.5'), Decimal(2.5), False, 3, "f", {}),
(1024, 1022, False, 2, "e", {}),
({"key": [Decimal('2.0001'), Decimal('20000.0001')]}, {"key": [2.0002, 20000.0002]}, True, 4, "e", {'values_changed': {"root['key'][0]": {'new_value': 2.0002, 'old_value': Decimal('2.0001')}}})
@pytest.mark.parametrize("test_num, t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result", [
(1, 43.265798602382986, 43.71677762295505, False, 0, "f", {'values_changed': {'root': {'new_value': 43.71677762295505, 'old_value': 43.265798602382986}}}), # Note that it rounds the number so one becomes 43 and the other one is 44
(2, Decimal('2.5'), Decimal('1.5'), False, 0, "f", {}),
(3, Decimal('2.5'), Decimal('1.5'), False, 1, "f", {'values_changed': {'root': {'new_value': Decimal('1.5'), 'old_value': Decimal('2.5')}}}),
(4, Decimal('2.5'), Decimal(2.5), False, 3, "f", {}),
(5, 1024, 1022, False, 2, "e", {}),
(6, {"key": [Decimal('2.0001'), Decimal('20000.0001')]}, {"key": [2.0002, 20000.0002]}, True, 4, "e", {'values_changed': {"root['key'][0]": {'new_value': 2.0002, 'old_value': Decimal('2.0001')}}}),
(7, [Decimal("999.99999999")], [Decimal("999.9999999999")], False, 6, "f", {}),
])
def test_significant_digits_and_notation(self, t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result):
def test_significant_digits_and_notation(self, test_num, t1, t2, ignore_numeric_type_changes, significant_digits, number_format_notation, result):
ddiff = DeepDiff(t1, t2, significant_digits=significant_digits, number_format_notation=number_format_notation,
ignore_numeric_type_changes=ignore_numeric_type_changes)
assert result == ddiff
assert result == ddiff, f"test_significant_digits_and_notation #{test_num} failed."

def test_significant_digits_for_complex_imaginary_part(self):
t1 = 1.23 + 1.222254j
@@ -1745,8 +1746,43 @@ def test_group_by2_when_repeats(self):
diff2 = DeepDiff(t1, t2, group_by='id', group_by_sort_key=lambda x: x['name'])
assert expected_grouped == diff2

diff3 = DeepDiff(t1, t2, group_by='id', group_by_sort_key=lambda x: x['name'])
assert expected_grouped == diff3
def test_group_by3_when_repeats_and_group_by_list(self):
t1 = [
{'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2},
{'id': 'BB', 'name': 'James', 'last_name': 'Blue', 'int_id': 20},
{'id': 'BB', 'name': 'Jimmy', 'last_name': 'Red', 'int_id': 3},
{'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4},
]

t2 = [
{'id': 'AA', 'name': 'Joe', 'last_name': 'Nobody', 'int_id': 2},
{'id': 'BB', 'name': 'James', 'last_name': 'Brown', 'int_id': 20},
{'id': 'CC', 'name': 'Mike', 'last_name': 'Apple', 'int_id': 4},
]

diff1 = DeepDiff(t1, t2, group_by=['id', 'name'])
expected_grouped = {
'dictionary_item_removed': ["root['BB']['Jimmy']"],
'values_changed': {
"root['BB']['James']['last_name']": {
'new_value': 'Brown',
'old_value': 'Blue'
}
}
}
assert expected_grouped == diff1

diff2 = DeepDiff(t1, t2, group_by=['id', 'name'], group_by_sort_key='int_id')
expected_grouped = {
'dictionary_item_removed': ["root['BB']['Jimmy']"],
'values_changed': {
"root['BB']['James'][0]['last_name']": {
'new_value': 'Brown',
'old_value': 'Blue'
}
}
}
assert expected_grouped == diff2

def test_group_by_key_missing(self):
t1 = [
15 changes: 9 additions & 6 deletions tests/test_serialization.py
Original file line number Diff line number Diff line change
@@ -315,13 +315,16 @@ def test_pretty_form_method(self, expected, verbose_level):
result = ddiff.pretty()
assert result == expected

@pytest.mark.parametrize('test_num, value', [
(1, {'10': None}),
(2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}}),
(3, {'10': Decimal(2017)}),
(4, Decimal(2017.1)),
@pytest.mark.parametrize('test_num, value, func_to_convert_back', [
(1, {'10': None}, None),
(2, {"type_changes": {"root": {"old_type": None, "new_type": list, "new_value": ["你好", 2, 3, 5]}}}, None),
(3, {'10': Decimal(2017)}, None),
(4, Decimal(2017.1), None),
(5, {1, 2, 10}, set),
])
def test_json_dumps_and_loads(self, test_num, value):
def test_json_dumps_and_loads(self, test_num, value, func_to_convert_back):
serialized = json_dumps(value)
back = json_loads(serialized)
if func_to_convert_back:
back = func_to_convert_back(back)
assert value == back, f"test_json_dumps_and_loads test #{test_num} failed"

0 comments on commit 84fcc41

Please sign in to comment.